Update theme and scan for dead links endpoint

GordonBeeming · May 31, 2024 · 6714aff · 6714aff
1 parent 902d59a
commit 6714aff
Show file tree

Hide file tree

Showing 19 changed files with 396 additions and 116 deletions.
diff --git a/src/GordonBeemingCom.Database/AppDbContext.cs b/src/GordonBeemingCom.Database/AppDbContext.cs
@@ -7,7 +7,7 @@ public partial class AppDbContext : DbContext
 {
 #pragma warning disable CS8618
   public AppDbContext(DbContextOptions<AppDbContext> options)
-        : base(options)
+    : base(options)
   {
   }
 
@@ -30,6 +30,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
     modelBuilder.Entity<AcceptedExternalUrls>(entity =>
     {
       entity.Property(e => e.DateTimeStamp).HasDefaultValueSql("(sysutcdatetime())");
+      entity.Property(e => e.LastCheckedDate).HasDefaultValue(new DateTime(2000, 1, 1));
 
       entity.HasQueryFilter(b => EF.Property<DateTime?>(b, "CancelledDate") == null);
     });
@@ -40,8 +41,8 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
       entity.Property(e => e.DateTimeStamp).HasDefaultValueSql("(sysutcdatetime())");
 
       entity.HasOne(d => d.Blog).WithMany(p => p.BlogContentBlocks)
-              .OnDelete(DeleteBehavior.ClientSetNull)
-              .HasConstraintName("FK_BlogContentBlocks_Blogs");
+        .OnDelete(DeleteBehavior.ClientSetNull)
+        .HasConstraintName("FK_BlogContentBlocks_Blogs");
 
       entity.HasQueryFilter(b => EF.Property<DateTime?>(b, "CancelledDate") == null);
     });
@@ -51,12 +52,12 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
       entity.Property(e => e.DateTimeStamp).HasDefaultValueSql("(sysutcdatetime())");
 
       entity.HasOne(d => d.Blog).WithMany(p => p.BlogTags)
-              .OnDelete(DeleteBehavior.ClientSetNull)
-              .HasConstraintName("FK_BlogTags_Blogs");
+        .OnDelete(DeleteBehavior.ClientSetNull)
+        .HasConstraintName("FK_BlogTags_Blogs");
 
       entity.HasOne(d => d.Tag).WithMany(p => p.BlogTags)
-              .OnDelete(DeleteBehavior.ClientSetNull)
-              .HasConstraintName("FK_BlogTags_Tags");
+        .OnDelete(DeleteBehavior.ClientSetNull)
+        .HasConstraintName("FK_BlogTags_Tags");
     });
 
     modelBuilder.Entity<Blogs>(entity =>
@@ -65,8 +66,8 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
       entity.Property(e => e.DateTimeStamp).HasDefaultValueSql("(sysutcdatetime())");
 
       entity.HasOne(d => d.Category).WithMany(p => p.Blogs)
-              .OnDelete(DeleteBehavior.ClientSetNull)
-              .HasConstraintName("FK_Blogs_Categories");
+        .OnDelete(DeleteBehavior.ClientSetNull)
+        .HasConstraintName("FK_Blogs_Categories");
 
       entity.HasQueryFilter(b => EF.Property<DateTime?>(b, "CancelledDate") == null);
     });
@@ -77,8 +78,8 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
       entity.Property(e => e.DateTimeStamp).HasDefaultValueSql("(getutcdate())");
 
       entity.HasOne(d => d.Blogs).WithMany(p => p.BlogsRedirectUrl)
-              .OnDelete(DeleteBehavior.ClientSetNull)
-              .HasConstraintName("FK_BlogsRedirectUrl_Blogs");
+        .OnDelete(DeleteBehavior.ClientSetNull)
+        .HasConstraintName("FK_BlogsRedirectUrl_Blogs");
     });
 
     modelBuilder.Entity<Categories>(entity =>

diff --git a/src/GordonBeemingCom.Database/Migrations/AppDbContextModelSnapshot.cs b/src/GordonBeemingCom.Database/Migrations/AppDbContextModelSnapshot.cs
@@ -17,7 +17,7 @@ protected override void BuildModel(ModelBuilder modelBuilder)
         {
 #pragma warning disable 612, 618
             modelBuilder
-                .HasAnnotation("ProductVersion", "9.0.0-preview.2.24128.4")
+                .HasAnnotation("ProductVersion", "9.0.0-preview.4.24267.1")
                 .HasAnnotation("Relational:MaxIdentifierLength", 128);
 
             SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder);
@@ -36,6 +36,28 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                         .HasColumnType("datetime2")
                         .HasDefaultValueSql("(sysutcdatetime())");
 
+                    b.Property<string>("DisableReason")
+                        .HasMaxLength(50)
+                        .HasColumnType("nvarchar(50)");
+
+                    b.Property<int>("ErrorCount")
+                        .HasColumnType("int");
+
+                    b.Property<string>("Headers")
+                        .IsRequired()
+                        .HasColumnType("nvarchar(max)");
+
+                    b.Property<int>("HttpStatusCode")
+                        .HasColumnType("int");
+
+                    b.Property<bool>("IsSuccessStatusCode")
+                        .HasColumnType("bit");
+
+                    b.Property<DateTime>("LastCheckedDate")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("datetime2")
+                        .HasDefaultValue(new DateTime(2000, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified));
+
                     b.Property<string>("Url")
                         .IsRequired()
                         .HasMaxLength(2048)

diff --git a/src/GordonBeemingCom.Database/Tables/AcceptedExternalUrls.cs b/src/GordonBeemingCom.Database/Tables/AcceptedExternalUrls.cs
@@ -1,21 +1,31 @@
 using System;
 using System.Collections.Generic;
+using System.ComponentModel;
 using System.ComponentModel.DataAnnotations;
 using System.ComponentModel.DataAnnotations.Schema;
+using System.Runtime.InteropServices.JavaScript;
 using Microsoft.EntityFrameworkCore;
 
 namespace GordonBeemingCom.Database.Tables;
 
 public partial class AcceptedExternalUrls
 {
-  [Key]
-  [StringLength(40)]
-  public required string UrlHash { get; set; }
+  [Key] [StringLength(40)] public required string UrlHash { get; set; }
 
-  [StringLength(2048)]
-  public required string Url { get; set; }
+  [Required] [StringLength(2048)] public required string Url { get; set; }
 
-  public DateTime DateTimeStamp { get; set; }
+  [Required] public DateTime DateTimeStamp { get; set; }
 
   public DateTime? CancelledDate { get; set; }
+
+  [StringLength(50)] public string? DisableReason { get; set; }
+
+  [Required] public required DateTime LastCheckedDate { get; set; }
+  [Required] public required int ErrorCount { get; set; }
+  [Required] public required int HttpStatusCode { get; set; }
+
+  // ReSharper disable once EntityFramework.ModelValidation.UnlimitedStringLength
+  [Required] public required string Headers { get; set; } = default!;
+
+  [Required] public bool IsSuccessStatusCode { get; set; }
 }
diff --git a/src/GordonBeemingCom.Editor/Endpoints/UpdateDeadLinks.cs b/src/GordonBeemingCom.Editor/Endpoints/UpdateDeadLinks.cs
@@ -0,0 +1,77 @@
+using System.Net;
+using System.Net.Http.Headers;
+using GordonBeemingCom.Shared.Models;
+using GordonBeemingCom.Shared.Services;
+using Microsoft.AspNetCore.Mvc;
+using NuGet.Protocol;
+
+namespace GordonBeemingCom.Editor.Endpoints;
+
+public static class ExternalUrlsServiceExtensions
+{
+  public static void RegisterUpdateDeadLinksEndpoint(this WebApplication app)
+  {
+    app.MapPost("/api/test", ()=> Results.Ok("Hello World"))
+      .AllowAnonymous();
+    app.MapPost("/api/update-dead-links", UpdateDeadLinks.EndPoint)
+      .AllowAnonymous();
+  }
+}
+
+public sealed class UpdateDeadLinks
+{
+  public static async Task<IResult> EndPoint(IExternalUrlsService externalUrlsService, ILogger<UpdateDeadLinks> logger,
+    IHttpClientFactory httpClientFactory)
+  {
+    var httpClient = httpClientFactory.CreateClient("link-checker");
+    var activeLinks = await externalUrlsService.GetActiveLinks(100);
+    var linksUpdated = 0;
+    foreach (var link in activeLinks)
+    {
+      try
+      {
+        // Use HEAD request to retrieve headers only
+        var request = new HttpRequestMessage(HttpMethod.Head, link.Url);
+        // Some sites... like unsplashed block bots, so we're pretend to be the Googlebot ??... surprisingly this works
+        // https://unsplash.com/@gordonbeeming
+        request.Headers.UserAgent.Add(new ProductInfoHeaderValue("Googlebot", "1.0"));
+        var response = await httpClient.SendAsync(request);
+
+        if (response.StatusCode == HttpStatusCode.Forbidden ||
+            response.StatusCode == HttpStatusCode.MethodNotAllowed ||
+            response.StatusCode == HttpStatusCode.TooManyRequests)
+        {
+          request = new HttpRequestMessage(HttpMethod.Get, link.Url);
+          response = await httpClient.SendAsync(request);
+        }
+
+        using var enumerator = response.Headers.GetEnumerator();
+        await externalUrlsService.UpdateLinkDetails(new ExternalLinkDetails
+        {
+          UrlHash = link.UrlHash,
+          Url = link.Url,
+          Headers = GetList(enumerator).ToList(),
+          IsSuccessStatusCode = response.IsSuccessStatusCode,
+          HttpStatusCode = (int)response.StatusCode,
+        });
+      }
+      catch (Exception e)
+      {
+        logger.LogError(e, "Error updating link [{Hash}] {Link}", link.UrlHash, link.Url);
+      }
+
+      linksUpdated++;
+    }
+
+    return Results.Ok(linksUpdated);
+  }
+
+  private static IEnumerable<KeyValuePair<string, List<string>>> GetList(
+    IEnumerator<KeyValuePair<string, IEnumerable<string>>> enumerator)
+  {
+    while (enumerator.MoveNext())
+    {
+      yield return new KeyValuePair<string, List<string>>(enumerator.Current.Key, enumerator.Current.Value.ToList());
+    }
+  }
+}
diff --git a/src/GordonBeemingCom.Editor/Pages/UpdateAcceptedUrls.razor b/src/GordonBeemingCom.Editor/Pages/UpdateAcceptedUrls.razor
@@ -9,11 +9,11 @@
 <PageTitle>Accepted Urls</PageTitle>
 
 <h1>Accepted Urls</h1>
-  
+
 <button type="submit" class="btn btn-primary" @onclick="ScanAllPosts">@(scanButtonText)</button>
 
 <EditForm Model="htmlContext">
-    
+
   <div class="row">
     <div class="form-group col-lg-12 col-sm-12">
       <label for="htmlInput">Html:</label>
@@ -55,7 +55,7 @@
   private int ScanCount { get; set; }
   private List<string> validationIssues = new();
   private List<string> discoveredLinks = new();
-  
+
   private HtmlContentBlockContext htmlContext = new HtmlContentBlockContext();
 
   protected override Task OnInitializedAsync()
@@ -110,17 +110,20 @@
   private async Task LoadDiscoveredLinks()
   {
     discoveredLinks = (await externalUrlsService.GetUrlsCacheAsync())
-      .OrderBy(o=>o)
+      .OrderBy(o => o.Url)
+      .Select(o => o.Url)
       .ToList();
     StateHasChanged();
   }
 
   private async Task ScanHtmlForUrls()
   {
+    // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract
     if (htmlContext is null)
     {
       return;
     }
+
     try
     {
       scanHtmlButtonText = "Scanning...";
@@ -141,4 +144,5 @@
       }
     }
   }
+
 }
diff --git a/src/GordonBeemingCom.Editor/Program.cs b/src/GordonBeemingCom.Editor/Program.cs
@@ -2,10 +2,12 @@
 using GordonBeemingCom.Database.Tables;
 using GordonBeemingCom.Editor.Areas.Identity;
 using GordonBeemingCom.Editor.Data;
+using GordonBeemingCom.Editor.Endpoints;
 using GordonBeemingCom.Shared.Services;
 using Microsoft.AspNetCore.Components;
 using Microsoft.AspNetCore.Components.Authorization;
 using Microsoft.AspNetCore.Components.Web;
+using Microsoft.AspNetCore.CookiePolicy;
 using Microsoft.AspNetCore.HttpOverrides;
 using Microsoft.AspNetCore.Identity;
 using Microsoft.AspNetCore.Identity.UI;
@@ -53,6 +55,20 @@
   })
     .AddEntityFrameworkStores<ApplicationDbContext>();
 
+builder.Services.AddAntiforgery(o =>
+{
+  o.HeaderName = "XSRF-TOKEN";
+  o.Cookie.HttpOnly = true;
+  o.Cookie.SecurePolicy = CookieSecurePolicy.Always;
+});
+
+builder.Services.Configure<CookiePolicyOptions>(options =>
+{
+  options.MinimumSameSitePolicy = SameSiteMode.Strict;
+  options.HttpOnly = HttpOnlyPolicy.Always;
+  options.Secure = CookieSecurePolicy.Always;
+});
+
 builder.Services.Configure<ForwardedHeadersOptions>(options =>
 {
   options.ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto;
@@ -76,6 +92,8 @@
 builder.Services.AddSingleton<DeploymentInfo>();
 builder.Services.AddScoped<IExternalUrlsService, ExternalUrlsService>();
 
+builder.Services.AddHttpClient();
+
 var app = builder.Build();
 
 var serviceScopeFactory = app.Services.GetService<IServiceScopeFactory>();
@@ -87,9 +105,9 @@
   {
     appDbContext.Categories.Add(new Categories() { Id = Guid.NewGuid(), CategoryName = "Developer", CategorySlug = "developer", DisplayIndex = 1, HexColour = "00FF00",});
     appDbContext.SaveChanges();
-
   }
 }
+app.RegisterUpdateDeadLinksEndpoint();
 
 // Configure the HTTP request pipeline.
 if (app.Environment.IsDevelopment())
@@ -111,6 +129,7 @@
 
 app.UseAuthorization();
 
+
 app.MapControllers();
 app.MapBlazorHub();
 app.MapFallbackToPage("/_Host");

diff --git a/src/GordonBeemingCom.Editor/wwwroot/css/admin.css b/src/GordonBeemingCom.Editor/wwwroot/css/admin.css
@@ -9,7 +9,7 @@ h1:focus {
 }
 
 a, .btn-link {
-  color: #CC4141;
+  color: #00BBFF;
 }
 
 .btn-primary {

diff --git a/src/GordonBeemingCom.Editor/wwwroot/images/background.jpg b/src/GordonBeemingCom.Editor/wwwroot/images/background.jpg
diff --git a/src/GordonBeemingCom.Shared/Models/ExternalLink.cs b/src/GordonBeemingCom.Shared/Models/ExternalLink.cs
@@ -0,0 +1,7 @@
+namespace GordonBeemingCom.Shared.Models;
+
+public sealed class ExternalLink
+{
+  public string UrlHash { get; set; } = default!;
+  public string Url { get; set; } = default!;
+}
diff --git a/src/GordonBeemingCom.Shared/Models/ExternalLinkDetails.cs b/src/GordonBeemingCom.Shared/Models/ExternalLinkDetails.cs
@@ -0,0 +1,10 @@
+namespace GordonBeemingCom.Shared.Models;
+
+public sealed record ExternalLinkDetails
+{
+  public string UrlHash { get; set; } = default!;
+  public string Url { get; set; } = default!;
+  public List<KeyValuePair<string, List<string>>> Headers { get; set; } = default!;
+  public bool IsSuccessStatusCode { get; set; } = default!;
+  public int HttpStatusCode { get; set; } = default!;
+}