diff --git a/.DS_Store b/.DS_Store index 041bb95b..a6974e2f 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/Damselfly.Core/Damselfly.Core.csproj b/Damselfly.Core/Damselfly.Core.csproj index 69b274b0..f5d10395 100644 --- a/Damselfly.Core/Damselfly.Core.csproj +++ b/Damselfly.Core/Damselfly.Core.csproj @@ -6,14 +6,14 @@ - - + + - + - + @@ -21,13 +21,13 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all diff --git a/Damselfly.Core/Migrations/20210115225147_HashIndex.Designer.cs b/Damselfly.Core/Migrations/20210115225147_HashIndex.Designer.cs new file mode 100644 index 00000000..f169b2cc --- /dev/null +++ b/Damselfly.Core/Migrations/20210115225147_HashIndex.Designer.cs @@ -0,0 +1,480 @@ +// +using System; +using Damselfly.Core.Models; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +namespace Damselfly.Core.Migrations +{ + [DbContext(typeof(ImageContext))] + [Migration("20210115225147_HashIndex")] + partial class HashIndex + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "5.0.2"); + + modelBuilder.Entity("Damselfly.Core.Models.Basket", b => + { + b.Property("BasketId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("DateAdded") + .HasColumnType("TEXT"); + + b.Property("Name") + .HasColumnType("TEXT"); + + b.HasKey("BasketId"); + + b.ToTable("Baskets"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.BasketEntry", b => + { + b.Property("BasketEntryId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("BasketId") + .HasColumnType("INTEGER"); + + b.Property("DateAdded") + .HasColumnType("TEXT"); + + b.Property("ImageId") + .HasColumnType("INTEGER"); + + b.HasKey("BasketEntryId"); + + b.HasIndex("BasketId"); + + b.HasIndex("ImageId") + .IsUnique(); + + b.ToTable("BasketEntries"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.Camera", b => + { + b.Property("CameraId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("Make") + .HasColumnType("TEXT"); + + b.Property("Model") + .HasColumnType("TEXT"); + + b.Property("Serial") + .HasColumnType("TEXT"); + + b.HasKey("CameraId"); + + b.ToTable("Cameras"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.ConfigSetting", b => + { + b.Property("ConfigSettingId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("Name") + .HasColumnType("TEXT"); + + b.Property("Value") + .HasColumnType("TEXT"); + + b.HasKey("ConfigSettingId"); + + b.ToTable("ConfigSettings"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.ExifOperation", b => + { + b.Property("ExifOperationId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ImageId") + .HasColumnType("INTEGER"); + + b.Property("Operation") + .HasColumnType("INTEGER"); + + b.Property("State") + .HasColumnType("INTEGER"); + + b.Property("Text") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("TimeStamp") + .HasColumnType("TEXT"); + + b.Property("Type") + .HasColumnType("INTEGER"); + + b.HasKey("ExifOperationId"); + + b.HasIndex("TimeStamp"); + + b.HasIndex("ImageId", "Text"); + + b.ToTable("KeywordOperations"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.ExportConfig", b => + { + b.Property("ExportConfigId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("Name") + .HasColumnType("TEXT"); + + b.Property("Size") + .HasColumnType("INTEGER"); + + b.Property("Type") + .HasColumnType("INTEGER"); + + b.Property("WatermarkText") + .HasColumnType("TEXT"); + + b.HasKey("ExportConfigId"); + + b.ToTable("DownloadConfigs"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.FTSTag", b => + { + b.Property("FTSTagId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("Keyword") + .HasColumnType("TEXT"); + + b.HasKey("FTSTagId"); + + b.ToTable("FTSTags"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.Folder", b => + { + b.Property("FolderId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("FolderScanDate") + .HasColumnType("TEXT"); + + b.Property("ParentFolderId") + .HasColumnType("INTEGER"); + + b.Property("Path") + .HasColumnType("TEXT"); + + b.HasKey("FolderId"); + + b.HasIndex("FolderScanDate"); + + b.HasIndex("Path"); + + b.ToTable("Folders"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.Image", b => + { + b.Property("ImageId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("FileCreationDate") + .HasColumnType("TEXT"); + + b.Property("FileLastModDate") + .HasColumnType("TEXT"); + + b.Property("FileName") + .HasColumnType("TEXT"); + + b.Property("FileSizeBytes") + .HasColumnType("INTEGER"); + + b.Property("FolderId") + .HasColumnType("INTEGER"); + + b.Property("LastUpdated") + .HasColumnType("TEXT"); + + b.Property("SortDate") + .HasColumnType("TEXT"); + + b.HasKey("ImageId"); + + b.HasIndex("FileLastModDate"); + + b.HasIndex("FileName"); + + b.HasIndex("FolderId"); + + b.HasIndex("LastUpdated"); + + b.HasIndex("SortDate"); + + b.ToTable("Images"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.ImageMetaData", b => + { + b.Property("MetaDataId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CameraId") + .HasColumnType("INTEGER"); + + b.Property("Caption") + .HasColumnType("TEXT"); + + b.Property("DateTaken") + .HasColumnType("TEXT"); + + b.Property("Description") + .HasColumnType("TEXT"); + + b.Property("Exposure") + .HasColumnType("TEXT"); + + b.Property("FNum") + .HasColumnType("TEXT"); + + b.Property("FlashFired") + .HasColumnType("INTEGER"); + + b.Property("Hash") + .HasColumnType("TEXT"); + + b.Property("Height") + .HasColumnType("INTEGER"); + + b.Property("ISO") + .HasColumnType("TEXT"); + + b.Property("ImageId") + .HasColumnType("INTEGER"); + + b.Property("LastUpdated") + .HasColumnType("TEXT"); + + b.Property("LensId") + .HasColumnType("INTEGER"); + + b.Property("Rating") + .HasColumnType("INTEGER"); + + b.Property("ThumbLastUpdated") + .HasColumnType("TEXT"); + + b.Property("Width") + .HasColumnType("INTEGER"); + + b.HasKey("MetaDataId"); + + b.HasIndex("CameraId"); + + b.HasIndex("DateTaken"); + + b.HasIndex("Hash"); + + b.HasIndex("ImageId") + .IsUnique(); + + b.HasIndex("LensId"); + + b.HasIndex("ThumbLastUpdated"); + + b.ToTable("ImageMetaData"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.ImageTag", b => + { + b.Property("ImageId") + .HasColumnType("INTEGER"); + + b.Property("TagId") + .HasColumnType("INTEGER"); + + b.HasKey("ImageId", "TagId"); + + b.HasIndex("TagId"); + + b.HasIndex("ImageId", "TagId") + .IsUnique(); + + b.ToTable("ImageTags"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.Lens", b => + { + b.Property("LensId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("Make") + .HasColumnType("TEXT"); + + b.Property("Model") + .HasColumnType("TEXT"); + + b.Property("Serial") + .HasColumnType("TEXT"); + + b.HasKey("LensId"); + + b.ToTable("Lenses"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.Tag", b => + { + b.Property("TagId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("Keyword") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("TimeStamp") + .HasColumnType("TEXT"); + + b.Property("Type") + .HasColumnType("TEXT"); + + b.HasKey("TagId"); + + b.HasIndex("Keyword") + .IsUnique(); + + b.ToTable("Tags"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.BasketEntry", b => + { + b.HasOne("Damselfly.Core.Models.Basket", "Basket") + .WithMany("BasketEntries") + .HasForeignKey("BasketId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Damselfly.Core.Models.Image", "Image") + .WithOne("BasketEntry") + .HasForeignKey("Damselfly.Core.Models.BasketEntry", "ImageId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Basket"); + + b.Navigation("Image"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.ExifOperation", b => + { + b.HasOne("Damselfly.Core.Models.Image", "Image") + .WithMany() + .HasForeignKey("ImageId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Image"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.Image", b => + { + b.HasOne("Damselfly.Core.Models.Folder", "Folder") + .WithMany("Images") + .HasForeignKey("FolderId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Folder"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.ImageMetaData", b => + { + b.HasOne("Damselfly.Core.Models.Camera", "Camera") + .WithMany() + .HasForeignKey("CameraId"); + + b.HasOne("Damselfly.Core.Models.Image", "Image") + .WithOne("MetaData") + .HasForeignKey("Damselfly.Core.Models.ImageMetaData", "ImageId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Damselfly.Core.Models.Lens", "Lens") + .WithMany() + .HasForeignKey("LensId"); + + b.Navigation("Camera"); + + b.Navigation("Image"); + + b.Navigation("Lens"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.ImageTag", b => + { + b.HasOne("Damselfly.Core.Models.Image", "Image") + .WithMany("ImageTags") + .HasForeignKey("ImageId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Damselfly.Core.Models.Tag", "Tag") + .WithMany("ImageTags") + .HasForeignKey("TagId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Image"); + + b.Navigation("Tag"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.Basket", b => + { + b.Navigation("BasketEntries"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.Folder", b => + { + b.Navigation("Images"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.Image", b => + { + b.Navigation("BasketEntry"); + + b.Navigation("ImageTags"); + + b.Navigation("MetaData"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.Tag", b => + { + b.Navigation("ImageTags"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/Damselfly.Core/Migrations/20210115225147_HashIndex.cs b/Damselfly.Core/Migrations/20210115225147_HashIndex.cs new file mode 100644 index 00000000..6cf8a4c6 --- /dev/null +++ b/Damselfly.Core/Migrations/20210115225147_HashIndex.cs @@ -0,0 +1,22 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +namespace Damselfly.Core.Migrations +{ + public partial class HashIndex : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateIndex( + name: "IX_ImageMetaData_Hash", + table: "ImageMetaData", + column: "Hash"); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropIndex( + name: "IX_ImageMetaData_Hash", + table: "ImageMetaData"); + } + } +} diff --git a/Damselfly.Core/Migrations/ImageContextModelSnapshot.cs b/Damselfly.Core/Migrations/ImageContextModelSnapshot.cs index 5b34867e..0e3d1166 100644 --- a/Damselfly.Core/Migrations/ImageContextModelSnapshot.cs +++ b/Damselfly.Core/Migrations/ImageContextModelSnapshot.cs @@ -14,7 +14,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) { #pragma warning disable 612, 618 modelBuilder - .HasAnnotation("ProductVersion", "5.0.1"); + .HasAnnotation("ProductVersion", "5.0.2"); modelBuilder.Entity("Damselfly.Core.Models.Basket", b => { @@ -292,6 +292,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasIndex("DateTaken"); + b.HasIndex("Hash"); + b.HasIndex("ImageId") .IsUnique(); diff --git a/Damselfly.Core/Models/ImageContext.cs b/Damselfly.Core/Models/ImageContext.cs index 2ac33f94..6ad219ca 100644 --- a/Damselfly.Core/Models/ImageContext.cs +++ b/Damselfly.Core/Models/ImageContext.cs @@ -74,6 +74,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) modelBuilder.Entity().HasIndex(x => new { x.Keyword }).IsUnique(); modelBuilder.Entity().HasIndex(x => x.ImageId); modelBuilder.Entity().HasIndex(x => x.DateTaken); + modelBuilder.Entity().HasIndex(x => x.Hash); modelBuilder.Entity().HasIndex(x => x.ThumbLastUpdated); modelBuilder.Entity().HasIndex(x => new { x.ImageId, x.Text }); modelBuilder.Entity().HasIndex(x => x.TimeStamp); diff --git a/Damselfly.Web/Damselfly.Web.csproj b/Damselfly.Web/Damselfly.Web.csproj index 0b382ca6..a48c18b2 100644 --- a/Damselfly.Web/Damselfly.Web.csproj +++ b/Damselfly.Web/Damselfly.Web.csproj @@ -56,8 +56,8 @@ - - + + diff --git a/Damselfly.Web/Data/ExportableImage.cs b/Damselfly.Web/Data/ExportableImage.cs index fdbb9b12..a129ef1c 100644 --- a/Damselfly.Web/Data/ExportableImage.cs +++ b/Damselfly.Web/Data/ExportableImage.cs @@ -4,12 +4,12 @@ namespace Damselfly.Web.Data { - public class ExportableImage + public class ListableImage { public string ThumbURL { get; private set; } public Image Image { get; private set; } - public ExportableImage(Image image, ThumbSize size) + public ListableImage(Image image, ThumbSize size) { Image = image; ThumbURL = ThumbnailService.Instance.GetThumbRequestPath(image, size, "/no-image.png"); diff --git a/Damselfly.Web/Data/ImageService.cs b/Damselfly.Web/Data/ImageService.cs index 3d720e44..b2b64052 100644 --- a/Damselfly.Web/Data/ImageService.cs +++ b/Damselfly.Web/Data/ImageService.cs @@ -1,6 +1,7 @@ using System; using System.Linq; using System.IO; +using System.Collections.Generic; using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; using Microsoft.AspNetCore.Mvc; @@ -123,6 +124,43 @@ public static Task GetImage(int imageId, bool includeMetadata = true ) return Task.FromResult(image); } + public static List> GetImagesWithDuplicates() + { + using var db = new ImageContext(); + var watch = new Stopwatch("GetImagesWithDupes"); + + // Craft the SQL manually as server-side groupby isn't supported by EF Core. + var sql = "SELECT im.* from ImageMetaData im where im.hash in (SELECT hash from ImageMetaData where hash is not null group by hash having count( distinct ImageID ) > 1)"; + + var dupeImageMetaData = db.ImageMetaData.FromSqlRaw(sql) + .Include(x => x.Image) + .ThenInclude(x => x.Folder) + .Select( x => x.Image ) + .ToList(); + + var listOfLists = dupeImageMetaData.Where( x => x.MetaData != null ) + .GroupBy(x => x.MetaData.Hash) + .Where( x => x != null ) + .Select( x => x.ToList() ) + .ToList(); + + return listOfLists; + } + + public static List GetImageDuplicates(Image image) + { + using var db = new ImageContext(); + var watch = new Stopwatch("GetImageDupes"); + + var dupes = db.ImageMetaData + .Where(x => x.Hash.Equals(image.MetaData.Hash) && x.ImageId != image.ImageId) + .Include( x => x.Image ) + .ThenInclude( x => x.Folder ) + .Select( x => x.Image ); + + return dupes.ToList(); + } + public static string GetImageThumbUrl(Image image, ThumbSize size) { string url = "/no-image.jpg"; diff --git a/Damselfly.Web/Pages/DuplicatesPage.razor b/Damselfly.Web/Pages/DuplicatesPage.razor new file mode 100644 index 00000000..3c7c1106 --- /dev/null +++ b/Damselfly.Web/Pages/DuplicatesPage.razor @@ -0,0 +1,72 @@ +@page "/image/duplicates" + +@inject NavigationService navContext +@inject ViewDataService ViewDataService + +@using Damselfly.Core.ImageProcessing + +
+
+
+ +

+ Images with Duplicates +

+
+
+
+ @if (imageLists != null) + { + @foreach (var list in imageLists) + { + + @foreach (var img in list) + { +
+ @img.FileName +
+ } + +
+ } + } + else + { +

Loading selection...

+ } +
+
+ +@code { + public Image CurrentImage { get; set; } + + List> imageLists = new List>(); + + public Task>> LoadData() + { + var watch = new Stopwatch("DupesLoadData"); + imageLists.Clear(); + + imageLists = ImageService.GetImagesWithDuplicates(); + watch.Stop(); + + return Task.FromResult(imageLists); + } + + + protected override async Task OnAfterRenderAsync(bool firstRender) + { + if (firstRender) + { + ViewDataService.ShowFolderList = false; + ViewDataService.ShowBasket = false; + ViewDataService.ShowExport = true; + ViewDataService.ShowTags = false; + + await LoadData(); + + StateHasChanged(); + } + } + +} diff --git a/Damselfly.Web/Pages/ExportPage.razor b/Damselfly.Web/Pages/ExportPage.razor index 70dd6bee..ac29ec97 100644 --- a/Damselfly.Web/Pages/ExportPage.razor +++ b/Damselfly.Web/Pages/ExportPage.razor @@ -44,14 +44,14 @@ @code { - readonly List images = new List(); + readonly List images = new List(); - public Task> LoadData() + public Task> LoadData() { var watch = new Stopwatch("ExportLoadData"); images.Clear(); - images.AddRange(basketService.SelectedImages.Select(x => new ExportableImage(x, ThumbSize.Small))); + images.AddRange(basketService.SelectedImages.Select(x => new ListableImage(x, ThumbSize.Small))); watch.Stop(); return Task.FromResult(images); diff --git a/Damselfly.Web/Pages/ImageDuplicatesPage.razor b/Damselfly.Web/Pages/ImageDuplicatesPage.razor new file mode 100644 index 00000000..110849a4 --- /dev/null +++ b/Damselfly.Web/Pages/ImageDuplicatesPage.razor @@ -0,0 +1,88 @@ +@page "/image/duplicates/{ImageID}" + +@inject NavigationService navContext +@inject ViewDataService ViewDataService + +@using Damselfly.Core.ImageProcessing + +
+
+
+ +

+ Duplicate Images for @CurrentImage.FileName +

+
+
+
+ @if (images != null) + { + @foreach (var img in images) + { +
+
+ +
+
+ @img.Image.FileName +
+
+ @img.Image.Folder.Name +
+
+ + } + } + else + { +

Loading selection...

+ } +
+
+ +@code { + [Parameter] + public string ImageID { get; set; } + + public Image CurrentImage { get; set; } + + readonly List images = new List(); + + protected override async Task OnParametersSetAsync() + { + if (Int32.TryParse(ImageID, out var imageId)) + { + CurrentImage = await ImageService.GetImage(imageId); + navContext.CurrentImage = CurrentImage; + } + } + + + public Task> LoadData() + { + var watch = new Stopwatch("DupesLoadData"); + images.Clear(); + + images.AddRange( ImageService.GetImageDuplicates(CurrentImage).Select( x => new ListableImage( x, ThumbSize.Small ) ) ); + watch.Stop(); + + return Task.FromResult(images); + } + + + protected override async Task OnAfterRenderAsync(bool firstRender) + { + if (firstRender) + { + ViewDataService.ShowFolderList = false; + ViewDataService.ShowBasket = false; + ViewDataService.ShowExport = true; + ViewDataService.ShowTags = false; + + await LoadData(); + + StateHasChanged(); + } + } + +} diff --git a/Damselfly.Web/Pages/ImagePage.razor b/Damselfly.Web/Pages/ImagePage.razor index d1251db3..ed0ee38a 100644 --- a/Damselfly.Web/Pages/ImagePage.razor +++ b/Damselfly.Web/Pages/ImagePage.razor @@ -1,5 +1,5 @@ @page "/image" -@page "/image/{ImageName}" +@page "/image/{ImageID}" @using Damselfly.Core.ImageProcessing @@ -48,7 +48,7 @@ @code { [Parameter] - public string ImageName { get; set; } + public string ImageID { get; set; } Image image; Image nextImage; @@ -86,8 +86,7 @@ protected async Task SetUpNavigation() { - int imageId = 0; - if (Int32.TryParse(ImageName, out imageId)) + if (Int32.TryParse(ImageID, out var imageId)) { image = await ImageService.GetImage(imageId); navContext.CurrentImage = image; diff --git a/Damselfly.Web/Shared/Images/TagList.razor b/Damselfly.Web/Shared/Images/TagList.razor index 1e86a6d2..16cb245b 100644 --- a/Damselfly.Web/Shared/Images/TagList.razor +++ b/Damselfly.Web/Shared/Images/TagList.razor @@ -15,7 +15,6 @@ {
- @@ -233,9 +232,4 @@ return searchList.Union(results, StringComparer.OrdinalIgnoreCase); } - - public string ConvertTag(Tag tag) - { - return tag?.Keyword; - } } diff --git a/Damselfly.Web/config/db/damselfly.db-wal b/Damselfly.Web/config/db/damselfly.db-wal deleted file mode 100644 index e69d1dc3..00000000 Binary files a/Damselfly.Web/config/db/damselfly.db-wal and /dev/null differ diff --git a/VERSION b/VERSION index 44a6f3fc..1892b926 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.49 +1.3.2