Metadata Fixes (#3533)

Co-authored-by: Midhun Sudhir <[email protected]>
Kareadita · Feb 6, 2025 · bb9621a · bb9621a
1 parent 40bbdcb
commit bb9621a
Show file tree

Hide file tree

Showing 25 changed files with 151 additions and 56 deletions.
diff --git a/API/Data/DataContext.cs b/API/Data/DataContext.cs
@@ -4,6 +4,7 @@
 using System.Text.Json;
 using System.Threading;
 using System.Threading.Tasks;
+using API.DTOs.KavitaPlus.Metadata;
 using API.Entities;
 using API.Entities.Enums;
 using API.Entities.Enums.UserPreferences;
@@ -217,13 +218,20 @@ protected override void OnModelCreating(ModelBuilder builder)
                 v => JsonSerializer.Serialize(v, JsonSerializerOptions.Default),
                 v => JsonSerializer.Deserialize<List<string>>(v, JsonSerializerOptions.Default)
             );
+        builder.Entity<MetadataSettings>()
+            .Property(x => x.Whitelist)
+            .HasConversion(
+                v => JsonSerializer.Serialize(v, JsonSerializerOptions.Default),
+                v => JsonSerializer.Deserialize<List<string>>(v, JsonSerializerOptions.Default)
+            );
 
         // Configure one-to-many relationship
         builder.Entity<MetadataSettings>()
             .HasMany(x => x.FieldMappings)
             .WithOne(x => x.MetadataSettings)
             .HasForeignKey(x => x.MetadataSettingsId)
             .OnDelete(DeleteBehavior.Cascade);
+
         builder.Entity<MetadataSettings>()
             .Property(b => b.Enabled)
             .HasDefaultValue(true);

diff --git a/API/Data/Repositories/ExternalSeriesMetadataRepository.cs b/API/Data/Repositories/ExternalSeriesMetadataRepository.cs
@@ -224,6 +224,7 @@ public async Task<IList<int>> GetAllSeriesIdsWithoutMetadata(int limit)
     public async Task<IList<ManageMatchSeriesDto>> GetAllSeries(ManageMatchFilterDto filter)
     {
         return await _context.Series
+            .Include(s => s.Library)
             .Where(s => !ExternalMetadataService.NonEligibleLibraryTypes.Contains(s.Library.Type))
             .FilterMatchState(filter.MatchStateOption)
             .OrderBy(s => s.NormalizedName)

diff --git a/API/Data/Seed.cs b/API/Data/Seed.cs
@@ -308,7 +308,7 @@ public static async Task SeedMetadataSettings(DataContext context)
                 EnableTags = false,
                 EnableGenres = true,
                 EnableLocalizedName = false,
-                FirstLastPeopleNaming = false,
+                FirstLastPeopleNaming = true,
                 PersonRoles = [PersonRole.Writer, PersonRole.CoverArtist, PersonRole.Character]
             };
             await context.MetadataSettings.AddAsync(existing);

diff --git a/API/Extensions/QueryExtensions/QueryableExtensions.cs b/API/Extensions/QueryExtensions/QueryableExtensions.cs
@@ -288,11 +288,15 @@ public static IQueryable<Series> FilterMatchState(this IQueryable<Series> query,
         return stateOption switch
         {
             MatchStateOption.All => query,
-            MatchStateOption.Matched => query.Where(s => s.ExternalSeriesMetadata != null && s.ExternalSeriesMetadata.ValidUntilUtc > DateTime.MinValue && !s.IsBlacklisted),
-            MatchStateOption.NotMatched => query.Where(s => (s.ExternalSeriesMetadata == null || s.ExternalSeriesMetadata.ValidUntilUtc == DateTime.MinValue) && !s.IsBlacklisted),
+            MatchStateOption.Matched => query
+                .Include(s => s.ExternalSeriesMetadata)
+                .Where(s => s.ExternalSeriesMetadata != null && s.ExternalSeriesMetadata.ValidUntilUtc > DateTime.MinValue && !s.IsBlacklisted),
+            MatchStateOption.NotMatched => query.
+                Include(s => s.ExternalSeriesMetadata)
+                .Where(s => (s.ExternalSeriesMetadata == null || s.ExternalSeriesMetadata.ValidUntilUtc == DateTime.MinValue) && !s.IsBlacklisted),
             MatchStateOption.Error => query.Where(s => s.IsBlacklisted),
             MatchStateOption.DontMatch => query.Where(s => s.DontMatch),
-            _ => throw new ArgumentOutOfRangeException(nameof(stateOption), stateOption, null)
+            _ => query
         };
     }
 }
diff --git a/API/Helpers/AutoMapperProfiles.cs b/API/Helpers/AutoMapperProfiles.cs
@@ -345,10 +345,13 @@ public AutoMapperProfiles()
                     opt.MapFrom(src => src))
             .ForMember(dest => dest.IsMatched,
                 opt =>
-                    opt.MapFrom(src => src.ExternalSeriesMetadata != null && src.ExternalSeriesMetadata.AniListId != 0 && src.ExternalSeriesMetadata.ValidUntilUtc > DateTime.MinValue))
+                    opt.MapFrom(src => src.ExternalSeriesMetadata != null && src.ExternalSeriesMetadata.AniListId != 0
+                                                                          && src.ExternalSeriesMetadata.ValidUntilUtc > DateTime.MinValue))
             .ForMember(dest => dest.ValidUntilUtc,
-                opt =>
-                    opt.MapFrom(src => src.ExternalSeriesMetadata.ValidUntilUtc));
+                opt => opt.MapFrom(src =>
+                    src.ExternalSeriesMetadata != null
+                        ? src.ExternalSeriesMetadata.ValidUntilUtc
+                        : DateTime.MinValue));
 
 
         CreateMap<MangaFile, FileExtensionExportDto>();
@@ -361,10 +364,14 @@ public AutoMapperProfiles()
             .ForMember(dest => dest.LibraryId, opt => opt.MapFrom(src => src.Volume.Series.LibraryId))
             .ForMember(dest => dest.LibraryType, opt => opt.MapFrom(src => src.Volume.Series.Library.Type));
 
+        CreateMap<MetadataFieldMapping, MetadataFieldMappingDto>();
+
         CreateMap<MetadataSettings, MetadataSettingsDto>()
             .ForMember(dest => dest.Blacklist, opt => opt.MapFrom(src => src.Blacklist ?? new List<string>()))
-            .ForMember(dest => dest.Whitelist, opt => opt.MapFrom(src => src.Whitelist ?? new List<string>()));
-        CreateMap<MetadataFieldMapping, MetadataFieldMappingDto>();
+            .ForMember(dest => dest.Whitelist, opt => opt.MapFrom(src => src.Whitelist ?? new List<string>()))
+            .ForMember(dest => dest.AgeRatingMappings, opt => opt.MapFrom(src => src.AgeRatingMappings ?? new Dictionary<string, AgeRating>()));
+
+
 
     }
 }
diff --git a/API/Services/Plus/ExternalMetadataService.cs b/API/Services/Plus/ExternalMetadataService.cs
@@ -195,6 +195,7 @@ public async Task<IList<ExternalSeriesMatchDto>> MatchSeries(MatchSeriesDto dto)
         var license = (await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.LicenseKey)).Value;
         var series = await _unitOfWork.SeriesRepository.GetSeriesByIdAsync(dto.SeriesId,
             SeriesIncludes.Metadata | SeriesIncludes.ExternalMetadata);
+        if (series == null) return [];
 
         var potentialAnilistId = ScrobblingService.ExtractId<int?>(dto.Query, ScrobblingService.AniListWeblinkWebsite);
         var potentialMalId = ScrobblingService.ExtractId<long?>(dto.Query, ScrobblingService.MalWeblinkWebsite);
@@ -512,7 +513,7 @@ private async Task<bool> WriteExternalMetadataToSeries(ExternalSeriesDetailDto e
             madeModification = true;
         }
 
-        if (settings.EnableStartDate && externalMetadata.StartDate.HasValue)
+        if (settings.EnableStartDate && !series.Metadata.ReleaseYearLocked && externalMetadata.StartDate.HasValue)
         {
             series.Metadata.ReleaseYear = externalMetadata.StartDate.Value.Year;
             madeModification = true;
@@ -526,7 +527,7 @@ private async Task<bool> WriteExternalMetadataToSeries(ExternalSeriesDetailDto e
         // Process Genres
         if (externalMetadata.Genres != null)
         {
-            foreach (var genre in externalMetadata.Genres.Where(g => !settings.Blacklist.Contains(g)))
+            foreach (var genre in externalMetadata.Genres)
             {
                 // Apply field mappings
                 var mappedGenre = ApplyFieldMapping(genre, MetadataFieldType.Genre, settings.FieldMappings);
@@ -537,9 +538,12 @@ private async Task<bool> WriteExternalMetadataToSeries(ExternalSeriesDetailDto e
             }
 
             // Strip blacklisted items from processedGenres
-            processedGenres = processedGenres.Distinct().Where(g => !settings.Blacklist.Contains(g)).ToList();
+            processedGenres = processedGenres
+                .Distinct()
+                .Where(g => !settings.Blacklist.Contains(g))
+                .ToList();
 
-            if (settings.EnableGenres && processedGenres.Count > 0)
+            if (settings.EnableGenres && !series.Metadata.GenresLocked && processedGenres.Count > 0)
             {
                 _logger.LogDebug("Found {GenreCount} genres for {SeriesName}", processedGenres.Count, series.Name);
                 var allGenres = (await _unitOfWork.GenreRepository.GetAllGenresByNamesAsync(processedGenres.Select(Parser.Normalize))).ToList();
@@ -567,13 +571,14 @@ private async Task<bool> WriteExternalMetadataToSeries(ExternalSeriesDetailDto e
             }
 
             // Strip blacklisted items from processedTags
-            processedTags = processedTags.Distinct()
+            processedTags = processedTags
+                .Distinct()
                 .Where(g => !settings.Blacklist.Contains(g))
                 .Where(g => settings.Whitelist.Count == 0 || settings.Whitelist.Contains(g))
                 .ToList();
 
             // Set the tags for the series and ensure they are in the DB
-            if (settings.EnableTags && processedTags.Count > 0)
+            if (settings.EnableTags && !series.Metadata.TagsLocked && processedTags.Count > 0)
             {
                 _logger.LogDebug("Found {TagCount} tags for {SeriesName}", processedTags.Count, series.Name);
                 var allTags = (await _unitOfWork.TagRepository.GetAllTagsByNameAsync(processedTags.Select(Parser.Normalize)))
@@ -591,22 +596,36 @@ private async Task<bool> WriteExternalMetadataToSeries(ExternalSeriesDetailDto e
 
         #region Age Rating
 
-        // Determine Age Rating
-        var ageRating = DetermineAgeRating(processedGenres.Concat(processedTags), settings.AgeRatingMappings);
-        if (!series.Metadata.AgeRatingLocked && series.Metadata.AgeRating <= ageRating)
+        if (!series.Metadata.AgeRatingLocked)
         {
-            series.Metadata.AgeRating = ageRating;
-            _unitOfWork.SeriesRepository.Update(series);
-            madeModification = true;
+            try
+            {
+                // Determine Age Rating
+                var totalTags = processedGenres
+                    .Concat(processedTags)
+                    .Concat(series.Metadata.Genres.Select(g => g.Title))
+                    .Concat(series.Metadata.Tags.Select(g => g.Title));
+
+                var ageRating = DetermineAgeRating(totalTags, settings.AgeRatingMappings);
+                if (!series.Metadata.AgeRatingLocked && series.Metadata.AgeRating <= ageRating)
+                {
+                    series.Metadata.AgeRating = ageRating;
+                    _unitOfWork.SeriesRepository.Update(series);
+                    madeModification = true;
+                }
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "There was an issue determining Age Rating for Series {SeriesName} ({SeriesId})", series.Name, series.Id);
+            }
         }
-
         #endregion
 
         #region People
 
         if (settings.EnablePeople)
         {
-            series.Metadata.People ??= new List<SeriesMetadataPeople>();
+            series.Metadata.People ??= [];
 
             // Ensure all people are named correctly
             externalMetadata.Staff = externalMetadata.Staff.Select(s =>
@@ -635,7 +654,10 @@ private async Task<bool> WriteExternalMetadataToSeries(ExternalSeriesDetailDto e
                     Name = w.Name,
                     AniListId = ScrobblingService.ExtractId<int>(w.Url, ScrobblingService.AniListStaffWebsite),
                     Description = CleanSummary(w.Description),
-                }).ToList();
+                })
+                .Concat(series.Metadata.People.Where(p => p.Role == PersonRole.Writer).Select(p => _mapper.Map<PersonDto>(p)))
+                .DistinctBy(p => Parser.Normalize(p.Name))
+                .ToList();
 
 
             // NOTE: PersonRoles can be a hashset
@@ -661,7 +683,10 @@ private async Task<bool> WriteExternalMetadataToSeries(ExternalSeriesDetailDto e
                     Name = w.Name,
                     AniListId = ScrobblingService.ExtractId<int>(w.Url, ScrobblingService.AniListStaffWebsite),
                     Description = CleanSummary(w.Description),
-                }).ToList();
+                })
+                .Concat(series.Metadata.People.Where(p => p.Role == PersonRole.CoverArtist).Select(p => _mapper.Map<PersonDto>(p)))
+                .DistinctBy(p => Parser.Normalize(p.Name))
+                .ToList();
 
             if (!series.Metadata.CoverArtistLocked && artists.Count > 0 &&  settings.PersonRoles.Contains(PersonRole.CoverArtist))
             {
@@ -684,7 +709,10 @@ private async Task<bool> WriteExternalMetadataToSeries(ExternalSeriesDetailDto e
                         Name = w.Name,
                         AniListId = ScrobblingService.ExtractId<int>(w.Url, ScrobblingService.AniListCharacterWebsite),
                         Description = CleanSummary(w.Description),
-                    }).ToList();
+                    })
+                    .Concat(series.Metadata.People.Where(p => p.Role == PersonRole.Character).Select(p => _mapper.Map<PersonDto>(p)))
+                    .DistinctBy(p => Parser.Normalize(p.Name))
+                    .ToList();
 
 
                 if (!series.Metadata.CharacterLocked && characters.Count > 0)
@@ -713,13 +741,27 @@ private async Task<bool> WriteExternalMetadataToSeries(ExternalSeriesDetailDto e
 
         #endregion
 
+        #region Publication Status
+
         if (!series.Metadata.PublicationStatusLocked && settings.EnablePublicationStatus)
         {
-            var chapters = (await _unitOfWork.SeriesRepository.GetSeriesByIdAsync(series.Id, SeriesIncludes.Chapters))!.Volumes.SelectMany(v => v.Chapters).ToList();
-            var wasChanged = DeterminePublicationStatus(series, chapters, externalMetadata);
-            _unitOfWork.SeriesRepository.Update(series);
-            madeModification = madeModification || wasChanged;
+            try
+            {
+                var chapters =
+                    (await _unitOfWork.SeriesRepository.GetSeriesByIdAsync(series.Id, SeriesIncludes.Chapters))!.Volumes
+                    .SelectMany(v => v.Chapters).ToList();
+                var wasChanged = DeterminePublicationStatus(series, chapters, externalMetadata);
+                _unitOfWork.SeriesRepository.Update(series);
+                madeModification = madeModification || wasChanged;
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "There was an issue determining Publication Status for Series {SeriesName} ({SeriesId})", series.Name, series.Id);
+            }
         }
+        #endregion
+
+        #region Relationships
 
         if (settings.EnableRelationships && externalMetadata.Relations != null && defaultAdmin != null)
         {
@@ -773,6 +815,7 @@ private async Task<bool> WriteExternalMetadataToSeries(ExternalSeriesDetailDto e
                 madeModification = true;
             }
         }
+        #endregion
 
         return madeModification;
     }
@@ -889,6 +932,8 @@ private bool DeterminePublicationStatus(Series series, List<Chapter> chapters, E
     private static AgeRating DetermineAgeRating(IEnumerable<string> values, Dictionary<string, AgeRating> mappings)
     {
         // Find highest age rating from mappings
+        mappings ??= new Dictionary<string, AgeRating>();
+
         return values
             .Select(v => mappings.TryGetValue(v, out var mapping) ? mapping : AgeRating.Unknown)
             .DefaultIfEmpty(AgeRating.Unknown)
@@ -913,6 +958,7 @@ private async Task<ExternalSeriesMetadata> GetOrCreateExternalSeriesMetadataForS
         };
         series.ExternalSeriesMetadata = externalSeriesMetadata;
         _unitOfWork.ExternalSeriesMetadataRepository.Attach(externalSeriesMetadata);
+
         return externalSeriesMetadata;
     }
 

diff --git a/API/Services/SeriesService.cs b/API/Services/SeriesService.cs
@@ -349,7 +349,8 @@ public static async Task HandlePeopleUpdateAsync(SeriesMetadata metadata, IColle
         var existingPeople = await unitOfWork.PersonRepository.GetPeopleByNames(normalizedNames);
 
         // Use a dictionary for quick lookups
-        var existingPeopleDictionary = existingPeople.DistinctBy(p => p.NormalizedName).ToDictionary(p => p.NormalizedName, p => p);
+        var existingPeopleDictionary = existingPeople.DistinctBy(p => p.NormalizedName)
+            .ToDictionary(p => p.NormalizedName, p => p);
 
         // List to track people that will be added to the metadata
         var peopleToAdd = new List<Person>();

diff --git a/API/Services/Tasks/ScannerService.cs b/API/Services/Tasks/ScannerService.cs
@@ -450,12 +450,12 @@ await _eventHub.SendMessageAsync(MessageFactory.Error,
             // That way logging and UI informing is all in one place with full context
             _logger.LogError("[ScannerService] Some of the root folders for the library are empty. " +
                              "Either your mount has been disconnected or you are trying to delete all series in the library. " +
-                             "Scan has be aborted. " +
+                             "Scan has been aborted. " +
                              "Check that your mount is connected or change the library's root folder and rescan");
 
             await _eventHub.SendMessageAsync(MessageFactory.Error, MessageFactory.ErrorEvent( $"Some of the root folders for the library, {libraryName}, are empty.",
                 "Either your mount has been disconnected or you are trying to delete all series in the library. " +
-                "Scan has be aborted. " +
+                "Scan has been aborted. " +
                 "Check that your mount is connected or change the library's root folder and rescan"));
 
             return false;

diff --git a/API/Services/Tasks/VersionUpdaterService.cs b/API/Services/Tasks/VersionUpdaterService.cs
@@ -114,6 +114,7 @@ private async Task EnrichWithNightlyInfo(List<UpdateNotificationDto> dtos)
 
                 var nightlyDto = new UpdateNotificationDto
                 {
+                    // TODO: I should pass Title to the FE so that Nightly Release can be localized
                     UpdateTitle = $"Nightly Release {nightly.Version} - {prInfo.Title}",
                     UpdateVersion = nightly.Version,
                     CurrentVersion = dto.CurrentVersion,
@@ -446,7 +447,7 @@ private static Dictionary<string, List<string>> ParseReleaseBody(string body)
     {
         var sections = new Dictionary<string, List<string>>(StringComparer.OrdinalIgnoreCase);
         var lines = body.Split('\n');
-        string currentSection = null;
+        string? currentSection = null;
 
         foreach (var line in lines)
         {

diff --git a/UI/Web/src/app/_models/series-detail/external-series-detail.ts b/UI/Web/src/app/_models/series-detail/external-series-detail.ts
@@ -37,6 +37,11 @@ export interface ExternalSeriesDetail {
   summary?: string;
   volumeCount?: number;
   chapterCount?: number;
+  /**
+   * These are duplicated with volumeCount based on where it's being invoked.
+   */
+  volumes?: number;
+  chapters?: number;
   staff: Array<SeriesStaff>;
   tags: Array<MetadataTagDto>;
   provider: ScrobbleProvider;

diff --git a/UI/Web/src/app/_single-module/details-tab/details-tab.component.html b/UI/Web/src/app/_single-module/details-tab/details-tab.component.html
@@ -63,7 +63,7 @@ <h4 class="header">{{t('tags-title')}}</h4>
   <div class="mb-3">
     <app-carousel-reel [items]="webLinks" [title]="t('weblinks-title')">
       <ng-template #carouselItem let-item>
-        <a class="me-1" [href]="item | safeHtml"  target="_blank" rel="noopener noreferrer" [title]="item">
+        <a class="me-1" [href]="item | safeUrl"  target="_blank" rel="noopener noreferrer" [title]="item">
           <app-image height="24px" width="24px" aria-hidden="true" [imageUrl]="imageService.getWebLinkImage(item)"
                      [errorImage]="imageService.errorWebLinkImage"></app-image>
         </a>

diff --git a/UI/Web/src/app/_single-module/details-tab/details-tab.component.ts b/UI/Web/src/app/_single-module/details-tab/details-tab.component.ts
@@ -22,6 +22,7 @@ import {SeriesFormatComponent} from "../../shared/series-format/series-format.co
 import {MangaFormatPipe} from "../../_pipes/manga-format.pipe";
 import {LanguageNamePipe} from "../../_pipes/language-name.pipe";
 import {AsyncPipe} from "@angular/common";
+import {SafeUrlPipe} from "../../_pipes/safe-url.pipe";
 
 @Component({
   selector: 'app-details-tab',
@@ -39,7 +40,8 @@ import {AsyncPipe} from "@angular/common";
     SeriesFormatComponent,
     MangaFormatPipe,
     LanguageNamePipe,
-    AsyncPipe
+    AsyncPipe,
+    SafeUrlPipe
   ],
   templateUrl: './details-tab.component.html',
   styleUrl: './details-tab.component.scss',