From 96993b151949491cb571b652ce0d70f1c1411288 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20G=C3=BCntert?= Date: Thu, 13 Oct 2022 22:19:56 +0200 Subject: [PATCH] Allow disabling IP masking (#14) * Added telemetry initializer adding client IP to custom dimensions * Made client IP key configurable --- .gitignore | 1 + ApplicationInsightsRequestLogging.sln | 7 ++++ README.md | 33 +++++++++++---- .../BodyLoggerMiddleware.cs | 20 ++++----- .../Extensions/ServiceCollectionExtensions.cs | 24 +++++------ .../Initializers/ClientIpInitializer.cs | 26 ++++++++++++ .../Options/BodyLoggerOptions.cs | 10 +++++ ...licationInsightsRequestLoggingTests.csproj | 1 + .../BodyLoggerMiddlewareTests.cs | 5 ++- .../FakeRemoteIpAddressMiddleware.cs | 14 +++++++ .../ManualTests/Controllers/TestController.cs | 22 ++++++++++ test/ManualTests/ManualTests.csproj | 18 ++++++++ test/ManualTests/Program.cs | 41 +++++++++++++++++++ .../Properties/launchSettings.json | 31 ++++++++++++++ test/ManualTests/appsettings.json | 12 ++++++ 15 files changed, 230 insertions(+), 35 deletions(-) create mode 100644 src/ApplicationInsightsRequestLogging/Initializers/ClientIpInitializer.cs create mode 100644 test/ApplicationInsightsRequestLoggingTests/FakeRemoteIpAddressMiddleware.cs create mode 100644 test/ManualTests/Controllers/TestController.cs create mode 100644 test/ManualTests/ManualTests.csproj create mode 100644 test/ManualTests/Program.cs create mode 100644 test/ManualTests/Properties/launchSettings.json create mode 100644 test/ManualTests/appsettings.json diff --git a/.gitignore b/.gitignore index f9ccc6f..18eda1d 100644 --- a/.gitignore +++ b/.gitignore @@ -349,3 +349,4 @@ MigrationBackup/ # Ionide (cross platform F# VS Code tools) working folder .ionide/ /.idea/ +/test/ManualTests/appsettings.Development.json diff --git a/ApplicationInsightsRequestLogging.sln b/ApplicationInsightsRequestLogging.sln index 120f957..e100a0f 100644 --- a/ApplicationInsightsRequestLogging.sln +++ b/ApplicationInsightsRequestLogging.sln @@ -20,6 +20,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{CD74FC7F-0 EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ApplicationInsightsRequestLoggingTests", "test\ApplicationInsightsRequestLoggingTests\ApplicationInsightsRequestLoggingTests.csproj", "{0CB48D95-C03F-4BD1-A4C0-DD1F7CC73F56}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ManualTests", "test\ManualTests\ManualTests.csproj", "{B711C0B7-E8FF-48E9-8A93-3A5DB113088E}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -34,6 +36,10 @@ Global {0CB48D95-C03F-4BD1-A4C0-DD1F7CC73F56}.Debug|Any CPU.Build.0 = Debug|Any CPU {0CB48D95-C03F-4BD1-A4C0-DD1F7CC73F56}.Release|Any CPU.ActiveCfg = Release|Any CPU {0CB48D95-C03F-4BD1-A4C0-DD1F7CC73F56}.Release|Any CPU.Build.0 = Release|Any CPU + {B711C0B7-E8FF-48E9-8A93-3A5DB113088E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B711C0B7-E8FF-48E9-8A93-3A5DB113088E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B711C0B7-E8FF-48E9-8A93-3A5DB113088E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B711C0B7-E8FF-48E9-8A93-3A5DB113088E}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -44,5 +50,6 @@ Global GlobalSection(NestedProjects) = preSolution {F88928B2-6AEA-485E-9AEE-0E0B3752302E} = {B1127F87-7877-4664-8E3E-9F86202A51DE} {0CB48D95-C03F-4BD1-A4C0-DD1F7CC73F56} = {CD74FC7F-0463-4FC1-8735-C9F430477CE8} + {B711C0B7-E8FF-48E9-8A93-3A5DB113088E} = {CD74FC7F-0463-4FC1-8735-C9F430477CE8} EndGlobalSection EndGlobal diff --git a/README.md b/README.md index 1d7fa51..5e55247 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ -# Extended HTTP request & response logging with Application Insights +# Extended request logging with Application Insights ## Introduction This nuget package provides a custom middleware that allows to write the body of an HTTP request/response to a custom dimension. -![](https://i.imgur.com/0fxsnKN.png) +![](https://i.imgur.com/CNbVKsx.png) ## Features @@ -13,7 +13,8 @@ This nuget package provides a custom middleware that allows to write the body of - Configure HTTP status code ranges that will trigger logging - Configure maximum body length to store - Provide optional cut off text -- Configure name of custom dimension key +- Configure name of custom dimension keys +- Disable IP masking without the need to modify the App Insights resource as described [here](https://learn.microsoft.com/en-us/azure/azure-monitor/app/ip-collection?tabs=net) > A word of warning! Writing the content of an HTTP body to Application Insights might reveal sensitive user information that otherwise would be hidden and protected in transfer via TLS. So use this with care and only during debugging or developing! @@ -36,14 +37,17 @@ public void ConfigureServices(IServiceCollection services) { // ... - services.AddApplicationInsightsTelemetry(Configuration["APPINSIGHTS_CONNECTIONSTRING"]); + // Register App Insights + services.AddApplicationInsightsTelemetry(); + + // Register this middleware services.AddAppInsightsHttpBodyLogging(); // ... } ``` -Finally configure the request pipeline. +Finally configure the request pipeline. Make sure the call to `UseAppInsightsHttpBodyLogging` happens as early as possible as the [order matters](https://docs.microsoft.com/en-us/aspnet/core/fundamentals/middleware/?view=aspnetcore-6.0#middleware-order). Have a look at this [issue](https://github.com/matthiasguentert/azure-appinsights-logger/issues/11) ```csharp public void Configure(IApplicationBuilder app, IWebHostEnvironment env) @@ -59,18 +63,19 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env) ``` ## Configuration -You can overwrite the default settings as follows: +You can overwrite the default settings as follows... ```csharp services.AddAppInsightsHttpBodyLogging(o => { o.HttpCodes.Add(StatusCodes.Status200OK); o.HttpVerbs.Add(HttpMethods.Get); - o.MaxBytes = 10000 - o.Appendix = "\nSNIP" + o.MaxBytes = 10000; + o.Appendix = "\nSNIP"; + o.DisableIpMasking = true; }); ``` -Or stick with the defaults which are defined in `BodyLoggerOptions`. +...or stick with the defaults which are defined in `BodyLoggerOptions`. ### BodyLoggerOptions @@ -108,6 +113,11 @@ public class BodyLoggerOptions /// public string ResponseBodyPropertyKey { get; set; } = "ResponseBody"; + /// + /// Which property key should be used + /// + public string ClientIpPropertyKey { get; set; } = "ClientIp"; + /// /// Defines the amount of bytes that should be read from HTTP context /// @@ -117,5 +127,10 @@ public class BodyLoggerOptions /// Defines the text to append in case the body should be truncated /// public string Appendix { get; set; } = "\n---8<------------------------\nTRUNCATED DUE TO MAXBYTES LIMIT"; + + /// + /// Controls storage of client IP addresses https://learn.microsoft.com/en-us/azure/azure-monitor/app/ip-collection?tabs=net + /// + public bool DisableIpMasking { get; set; } = false; } ``` diff --git a/src/ApplicationInsightsRequestLogging/BodyLoggerMiddleware.cs b/src/ApplicationInsightsRequestLogging/BodyLoggerMiddleware.cs index 44a00ab..843905f 100644 --- a/src/ApplicationInsightsRequestLogging/BodyLoggerMiddleware.cs +++ b/src/ApplicationInsightsRequestLogging/BodyLoggerMiddleware.cs @@ -7,13 +7,13 @@ namespace Azureblue.ApplicationInsights.RequestLogging { public class BodyLoggerMiddleware : IMiddleware { - private readonly IOptions _options; + private readonly BodyLoggerOptions _options; private readonly IBodyReader _bodyReader; private readonly ITelemetryWriter _telemetryWriter; public BodyLoggerMiddleware(IOptions options, IBodyReader bodyReader, ITelemetryWriter telemetryWriter) { - _options = options ?? throw new ArgumentNullException(nameof(options)); + _options = options.Value ?? throw new ArgumentNullException(nameof(options)); _bodyReader = bodyReader ?? throw new ArgumentNullException(nameof(bodyReader)); _telemetryWriter = telemetryWriter ?? throw new ArgumentNullException(nameof(telemetryWriter)); } @@ -21,10 +21,10 @@ public BodyLoggerMiddleware(IOptions options, IBodyReader bod public async Task InvokeAsync(HttpContext context, RequestDelegate next) { var requestBody = string.Empty; - if (_options.Value.HttpVerbs.Contains(context.Request.Method)) + if (_options.HttpVerbs.Contains(context.Request.Method)) { // Temporarily store request body - requestBody = await _bodyReader.ReadRequestBodyAsync(context, _options.Value.MaxBytes, _options.Value.Appendix); + requestBody = await _bodyReader.ReadRequestBodyAsync(context, _options.MaxBytes, _options.Appendix); _bodyReader.PrepareResponseBodyReading(context); } @@ -32,19 +32,19 @@ public async Task InvokeAsync(HttpContext context, RequestDelegate next) // hand over to the next middleware and wait for the call to return await next(context); - if (_options.Value.HttpVerbs.Contains(context.Request.Method)) + if (_options.HttpVerbs.Contains(context.Request.Method)) { - if (_options.Value.HttpCodes.Contains(context.Response.StatusCode)) + if (_options.HttpCodes.Contains(context.Response.StatusCode)) { - var responseBody = await _bodyReader.ReadResponseBodyAsync(context, _options.Value.MaxBytes, _options.Value.Appendix); + var responseBody = await _bodyReader.ReadResponseBodyAsync(context, _options.MaxBytes, _options.Appendix); - _telemetryWriter.Write(context, _options.Value.RequestBodyPropertyKey, requestBody); - _telemetryWriter.Write(context, _options.Value.ResponseBodyPropertyKey, responseBody); + _telemetryWriter.Write(context, _options.RequestBodyPropertyKey, requestBody); + _telemetryWriter.Write(context, _options.ResponseBodyPropertyKey, responseBody); } // Copy back so response body is available for the user agent // prevent 500 error when Not StatusCode of Interest - await this._bodyReader.RestoreOriginalResponseBodyStreamAsync(context); + await _bodyReader.RestoreOriginalResponseBodyStreamAsync(context); } } } diff --git a/src/ApplicationInsightsRequestLogging/Extensions/ServiceCollectionExtensions.cs b/src/ApplicationInsightsRequestLogging/Extensions/ServiceCollectionExtensions.cs index 2725094..9ffcff9 100644 --- a/src/ApplicationInsightsRequestLogging/Extensions/ServiceCollectionExtensions.cs +++ b/src/ApplicationInsightsRequestLogging/Extensions/ServiceCollectionExtensions.cs @@ -1,5 +1,6 @@ using Microsoft.Extensions.DependencyInjection; using System; +using Microsoft.ApplicationInsights.Extensibility; namespace Azureblue.ApplicationInsights.RequestLogging { @@ -8,9 +9,7 @@ public static class ServiceCollectionExtensions public static IServiceCollection AddAppInsightsHttpBodyLogging(this IServiceCollection services) { if (services == null) - { throw new ArgumentNullException(nameof(services)); - } services.AddOptions(); AddBodyLogger(services); @@ -21,31 +20,28 @@ public static IServiceCollection AddAppInsightsHttpBodyLogging(this IServiceColl public static IServiceCollection AddAppInsightsHttpBodyLogging(this IServiceCollection services, Action setupAction) { if (services == null) - { throw new ArgumentNullException(nameof(services)); - } if (setupAction == null) - { throw new ArgumentNullException(nameof(setupAction)); - } AddBodyLogger(services, setupAction); return services; } - internal static void AddBodyLogger(IServiceCollection services) - { - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - } - - internal static void AddBodyLogger(IServiceCollection services, Action setupAction) + private static void AddBodyLogger(IServiceCollection services, Action setupAction) { AddBodyLogger(services); services.Configure(setupAction); } + + private static void AddBodyLogger(IServiceCollection services) + { + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + } } } diff --git a/src/ApplicationInsightsRequestLogging/Initializers/ClientIpInitializer.cs b/src/ApplicationInsightsRequestLogging/Initializers/ClientIpInitializer.cs new file mode 100644 index 0000000..3e7012d --- /dev/null +++ b/src/ApplicationInsightsRequestLogging/Initializers/ClientIpInitializer.cs @@ -0,0 +1,26 @@ +using Microsoft.ApplicationInsights.Channel; +using Microsoft.ApplicationInsights.DataContracts; +using Microsoft.ApplicationInsights.Extensibility; +using Microsoft.Extensions.Options; + +namespace Azureblue.ApplicationInsights.RequestLogging +{ + public class ClientIpInitializer : ITelemetryInitializer + { + private readonly BodyLoggerOptions _options; + + public ClientIpInitializer(IOptions options) => _options = options.Value; + + public void Initialize(ITelemetry telemetry) + { + if (!_options.DisableIpMasking) return; + + var clientIpKey = _options.ClientIpPropertyKey; + if (telemetry is ISupportProperties propTelemetry && !propTelemetry.Properties.ContainsKey(clientIpKey)) + { + var clientIpValue = telemetry.Context.Location.Ip; + propTelemetry.Properties.Add(clientIpKey, clientIpValue); + } + } + } +} \ No newline at end of file diff --git a/src/ApplicationInsightsRequestLogging/Options/BodyLoggerOptions.cs b/src/ApplicationInsightsRequestLogging/Options/BodyLoggerOptions.cs index 6b73385..ef5591b 100644 --- a/src/ApplicationInsightsRequestLogging/Options/BodyLoggerOptions.cs +++ b/src/ApplicationInsightsRequestLogging/Options/BodyLoggerOptions.cs @@ -36,6 +36,11 @@ public BodyLoggerOptions() /// public string ResponseBodyPropertyKey { get; set; } = "ResponseBody"; + /// + /// Which property key should be used + /// + public string ClientIpPropertyKey { get; set; } = "ClientIp"; + /// /// Defines the amount of bytes that should be read from HTTP context /// @@ -45,5 +50,10 @@ public BodyLoggerOptions() /// Defines the text to append in case the body should be truncated /// public string Appendix { get; set; } = "\n---8<------------------------\nTRUNCATED DUE TO MAXBYTES LIMIT"; + + /// + /// Controls storage of client IP addresses https://learn.microsoft.com/en-us/azure/azure-monitor/app/ip-collection?tabs=net + /// + public bool DisableIpMasking { get; set; } = false; } } diff --git a/test/ApplicationInsightsRequestLoggingTests/ApplicationInsightsRequestLoggingTests.csproj b/test/ApplicationInsightsRequestLoggingTests/ApplicationInsightsRequestLoggingTests.csproj index 4ba30d7..188a071 100644 --- a/test/ApplicationInsightsRequestLoggingTests/ApplicationInsightsRequestLoggingTests.csproj +++ b/test/ApplicationInsightsRequestLoggingTests/ApplicationInsightsRequestLoggingTests.csproj @@ -8,6 +8,7 @@ + diff --git a/test/ApplicationInsightsRequestLoggingTests/BodyLoggerMiddlewareTests.cs b/test/ApplicationInsightsRequestLoggingTests/BodyLoggerMiddlewareTests.cs index 8574a61..627bb32 100644 --- a/test/ApplicationInsightsRequestLoggingTests/BodyLoggerMiddlewareTests.cs +++ b/test/ApplicationInsightsRequestLoggingTests/BodyLoggerMiddlewareTests.cs @@ -1,6 +1,7 @@ using Azureblue.ApplicationInsights.RequestLogging; using FluentAssertions; using System; +using System.Linq; using Xunit; using Microsoft.AspNetCore.TestHost; using Microsoft.Extensions.Hosting; @@ -19,10 +20,10 @@ public class BodyLoggerMiddlewareTests public void BodyLoggerMiddleware_Should_Throw_If_Ctor_Params_Null() { // Arrange & Act - Action action = () => { var middleware = new BodyLoggerMiddleware(null, null, null); }; + var action = () => { var middleware = new BodyLoggerMiddleware(null, null, null); }; // Assert - action.Should().Throw(); + action.Should().Throw(); } [Fact] diff --git a/test/ApplicationInsightsRequestLoggingTests/FakeRemoteIpAddressMiddleware.cs b/test/ApplicationInsightsRequestLoggingTests/FakeRemoteIpAddressMiddleware.cs new file mode 100644 index 0000000..6dd2ae5 --- /dev/null +++ b/test/ApplicationInsightsRequestLoggingTests/FakeRemoteIpAddressMiddleware.cs @@ -0,0 +1,14 @@ +using System.Net; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; + +namespace ApplicationInsightsRequestLoggingTests; + +public class FakeRemoteIpAddressMiddleware : IMiddleware +{ + public async Task InvokeAsync(HttpContext context, RequestDelegate next) + { + context.Connection.RemoteIpAddress = IPAddress.Parse("127.168.1.32"); + await next(context); + } +} \ No newline at end of file diff --git a/test/ManualTests/Controllers/TestController.cs b/test/ManualTests/Controllers/TestController.cs new file mode 100644 index 0000000..bdc1397 --- /dev/null +++ b/test/ManualTests/Controllers/TestController.cs @@ -0,0 +1,22 @@ +using Microsoft.AspNetCore.Mvc; + +namespace ManualTests.Controllers +{ + [Route("api/")] + [ApiController] + public class TestController : ControllerBase + { + [HttpPost("test")] + public ActionResult Test([FromBody] TestData data) + { + return Ok(data); + } + } +} + +public class TestData +{ + public string Name { get; set; } + public string Blog { get; set; } + public string Topics { get; set; } +} \ No newline at end of file diff --git a/test/ManualTests/ManualTests.csproj b/test/ManualTests/ManualTests.csproj new file mode 100644 index 0000000..b73d7c2 --- /dev/null +++ b/test/ManualTests/ManualTests.csproj @@ -0,0 +1,18 @@ + + + + net6.0 + enable + enable + + + + + + + + + + + + diff --git a/test/ManualTests/Program.cs b/test/ManualTests/Program.cs new file mode 100644 index 0000000..a0c79fc --- /dev/null +++ b/test/ManualTests/Program.cs @@ -0,0 +1,41 @@ +using Azureblue.ApplicationInsights.RequestLogging; + +var builder = WebApplication.CreateBuilder(args); + +ConfigureConfiguration(builder.Configuration); +ConfigureServices(builder.Services); + +var app = builder.Build(); + +ConfigureMiddleware(app, app.Services); +ConfigureEndpoints(app, app.Services); + +app.Run(); + +void ConfigureConfiguration(ConfigurationManager configuration) {} +void ConfigureServices(IServiceCollection services) +{ + services.AddApplicationInsightsTelemetry(); + services.AddAppInsightsHttpBodyLogging(o => + { + o.HttpCodes.Add(StatusCodes.Status200OK); + o.DisableIpMasking = true; + }); + services.AddControllers(); + services.AddEndpointsApiExplorer(); + services.AddSwaggerGen(); +} + +void ConfigureMiddleware(IApplicationBuilder app, IServiceProvider services) +{ + app.UseAppInsightsHttpBodyLogging(); + app.UseSwagger(); + app.UseSwaggerUI(); + app.UseHttpsRedirection(); + app.UseAuthorization(); +} + +void ConfigureEndpoints(IEndpointRouteBuilder app, IServiceProvider services) +{ + app.MapControllers(); +} \ No newline at end of file diff --git a/test/ManualTests/Properties/launchSettings.json b/test/ManualTests/Properties/launchSettings.json new file mode 100644 index 0000000..b90b783 --- /dev/null +++ b/test/ManualTests/Properties/launchSettings.json @@ -0,0 +1,31 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "iisSettings": { + "windowsAuthentication": false, + "anonymousAuthentication": true, + "iisExpress": { + "applicationUrl": "http://localhost:4693", + "sslPort": 44300 + } + }, + "profiles": { + "WebApplication1": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "launchUrl": "swagger", + "applicationUrl": "https://localhost:7180;http://localhost:5178", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "IIS Express": { + "commandName": "IISExpress", + "launchBrowser": true, + "launchUrl": "swagger", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/test/ManualTests/appsettings.json b/test/ManualTests/appsettings.json new file mode 100644 index 0000000..732b0ef --- /dev/null +++ b/test/ManualTests/appsettings.json @@ -0,0 +1,12 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "AllowedHosts": "*", + "ApplicationInsights": { + "ConnectionString": "replace-me" + } +}