feat: database sink

GZTimeWalker · Dec 29, 2023 · aba9115 · aba9115
1 parent 5061337
commit aba9115
Show file tree

Hide file tree

Showing 3 changed files with 93 additions and 48 deletions.
diff --git a/src/GZCTF/Extensions/DatabaseSinkExtension.cs b/src/GZCTF/Extensions/DatabaseSinkExtension.cs
@@ -0,0 +1,68 @@
+using Serilog;
+using Serilog.Configuration;
+using Serilog.Core;
+using Serilog.Events;
+
+namespace GZCTF.Extensions;
+
+public static class DatabaseSinkExtension
+{
+    public static LoggerConfiguration Database(this LoggerSinkConfiguration loggerConfiguration,
+        IServiceProvider serviceProvider) =>
+        loggerConfiguration.Sink(new DatabaseSink(serviceProvider));
+}
+
+public class DatabaseSink : ILogEventSink
+{
+    readonly IServiceProvider _serviceProvider;
+
+    static DateTimeOffset LastFlushTime = DateTimeOffset.Now;
+    static readonly List<LogModel> LockedLogBuffer = new();
+    static readonly List<LogModel> LogBuffer = new();
+
+    public DatabaseSink(IServiceProvider serviceProvider)
+    {
+        _serviceProvider = serviceProvider;
+    }
+
+    public void Emit(LogEvent logEvent)
+    {
+        if (logEvent.Level < LogEventLevel.Information) return;
+
+        LogModel logModel = new()
+        {
+            TimeUTC = logEvent.Timestamp.ToUniversalTime(),
+            Level = logEvent.Level.ToString(),
+            Message = logEvent.RenderMessage(),
+            UserName = logEvent.Properties["UserName"].ToString()[1..^1],
+            Logger = logEvent.Properties["SourceContext"].ToString()[1..^1],
+            RemoteIP = logEvent.Properties["IP"].ToString()[1..^1],
+            Status = logEvent.Properties["Status"].ToString(),
+            Exception = logEvent.Exception?.ToString()
+        };
+
+        lock (LogBuffer)
+        {
+            LogBuffer.Add(logModel);
+
+            var needFlush = DateTimeOffset.Now - LastFlushTime > TimeSpan.FromSeconds(10);
+            if (!needFlush && LogBuffer.Count < 100) return;
+
+            LockedLogBuffer.AddRange(LogBuffer);
+            LogBuffer.Clear();
+
+            Task.Run(Flush);
+        }
+    }
+
+    async Task Flush()
+    {
+        using var scope = _serviceProvider.CreateScope();
+        var dbContext = scope.ServiceProvider.GetRequiredService<AppDbContext>();
+        await dbContext.Logs.AddRangeAsync(LockedLogBuffer);
+        await dbContext.SaveChangesAsync();
+
+        LockedLogBuffer.Clear();
+        LastFlushTime = DateTimeOffset.Now;
+    }
+}
diff --git a/src/GZCTF/Extensions/SignalRSinkExtension.cs b/src/GZCTF/Extensions/SignalRSinkExtension.cs
@@ -28,25 +28,26 @@ public SignalRSink(IServiceProvider serviceProvider)
 
     public void Emit(LogEvent logEvent)
     {
+        if (logEvent.Level < LogEventLevel.Information) return;
+
         _hubContext ??= _serviceProvider.GetRequiredService<IHubContext<AdminHub, IAdminClient>>();
 
-        if (logEvent.Level >= LogEventLevel.Information)
-            try
-            {
-                _hubContext.Clients.All.ReceivedLog(
-                    new LogMessageModel
-                    {
-                        Time = logEvent.Timestamp,
-                        Level = logEvent.Level.ToString(),
-                        UserName = logEvent.Properties["UserName"].ToString()[1..^1],
-                        IP = logEvent.Properties["IP"].ToString()[1..^1],
-                        Msg = logEvent.RenderMessage(),
-                        Status = logEvent.Properties["Status"].ToString()
-                    }).Wait();
-            }
-            catch
-            {
-                // ignored
-            }
+        try
+        {
+            _hubContext.Clients.All.ReceivedLog(
+                new LogMessageModel
+                {
+                    Time = logEvent.Timestamp,
+                    Level = logEvent.Level.ToString(),
+                    UserName = logEvent.Properties["UserName"].ToString()[1..^1],
+                    IP = logEvent.Properties["IP"].ToString()[1..^1],
+                    Msg = logEvent.RenderMessage(),
+                    Status = logEvent.Properties["Status"].ToString()
+                }).Wait();
+        }
+        catch
+        {
+            // ignored
+        }
     }
 }
diff --git a/src/GZCTF/Utils/LogHelper.cs b/src/GZCTF/Utils/LogHelper.cs
@@ -1,12 +1,10 @@
 using System.IO.Compression;
 using System.Net;
 using GZCTF.Extensions;
-using NpgsqlTypes;
 using Serilog;
 using Serilog.Events;
 using Serilog.Filters;
 using Serilog.Sinks.File.Archive;
-using Serilog.Sinks.PostgreSQL;
 using Serilog.Templates;
 using Serilog.Templates.Themes;
 using ILogger = Serilog.ILogger;
@@ -15,23 +13,13 @@ namespace GZCTF.Utils;
 
 public static class LogHelper
 {
-    const string LogTemplate =
-        "[{@t:yy-MM-dd HH:mm:ss.fff} {@l:u3}] {Substring(SourceContext, LastIndexOf(SourceContext, '.') + 1)}: {@m} {#if Length(Status) > 0}#{Status} <{UserName}>{#if Length(IP) > 0}@{IP}{#end}{#end}\n{@x}";
+    const string LogTemplate = "[{@t:yy-MM-dd HH:mm:ss.fff} {@l:u3}] " +
+                               "{Substring(SourceContext, LastIndexOf(SourceContext, '.') + 1)}: " +
+                               "{@m} {#if Length(Status) > 0}#{Status} <{UserName}>" +
+                               "{#if Length(IP) > 0}@{IP}{#end}{#end}\n{@x}";
 
     const string InitLogTemplate = "[{@t:yy-MM-dd HH:mm:ss.fff} {@l:u3}] {@m}\n{@x}";
 
-    static IDictionary<string, ColumnWriterBase> ColumnWriters => new Dictionary<string, ColumnWriterBase>
-    {
-        { "Message", new RenderedMessageColumnWriter() },
-        { "Level", new LevelColumnWriter(true, NpgsqlDbType.Varchar) },
-        { "TimeUTC", new TimeColumnWriter() },
-        { "Exception", new ExceptionColumnWriter() },
-        { "Logger", new SinglePropertyColumnWriter("SourceContext", PropertyWriteMethod.Raw, NpgsqlDbType.Varchar) },
-        { "UserName", new SinglePropertyColumnWriter("UserName", PropertyWriteMethod.Raw, NpgsqlDbType.Varchar) },
-        { "Status", new SinglePropertyColumnWriter("Status", PropertyWriteMethod.ToString, NpgsqlDbType.Varchar) },
-        { "RemoteIP", new SinglePropertyColumnWriter("IP", PropertyWriteMethod.Raw, NpgsqlDbType.Varchar) }
-    };
-
     /// <summary>
     /// 记录一条系统日志（无用户信息，默认Info）
     /// </summary>
@@ -97,7 +85,7 @@ public static void Log<T>(this ILogger<T> logger, string msg, string uname, stri
     {
         using (logger.BeginScope("{UserName}{Status}{IP}", uname, status, ip))
         {
-            logger.Log(level ?? LogLevel.Information, "{msg}", msg);
+            logger.Log(level ?? LogLevel.Information, "{msg:l}", msg);
         }
     }
 
@@ -160,19 +148,7 @@ public static ILogger GetLogger(IConfiguration configuration, IServiceProvider s
                 retainedFileCountLimit: 5,
                 hooks: new ArchiveHooks(CompressionLevel.Optimal, $"{FilePath.Logs}/archive/{{UtcDate:yyyy-MM}}")
             ))
-            .WriteTo.Async(t => t.PostgreSQL(
-                configuration.GetConnectionString("Database"),
-                "Logs",
-                respectCase: true,
-                columnOptions: ColumnWriters,
-                restrictedToMinimumLevel: LogEventLevel.Information,
-                period: TimeSpan.FromSeconds(30)
-            ))
+            .WriteTo.Database(serviceProvider)
             .WriteTo.SignalR(serviceProvider)
             .CreateLogger();
-}
-
-public class TimeColumnWriter() : ColumnWriterBase(NpgsqlDbType.TimestampTz)
-{
-    public override object GetValue(LogEvent logEvent, IFormatProvider? formatProvider = null) => logEvent.Timestamp.ToUniversalTime();
 }