From 875a391b198b3635a6b9ca42e3918c41e6a646ac Mon Sep 17 00:00:00 2001 From: Alexander Nikolaev Date: Mon, 18 May 2020 17:09:26 +0200 Subject: [PATCH 01/30] Request can be affinitized to destinations by an affinity key extracted from cookie or custom header --- .../BackendDiscovery/Contract/Backend.cs | 7 ++ .../Contract/SessionAffinityMode.cs | 18 ++++ .../Contract/SessionAffinityOptions.cs | 24 +++++ .../IReverseProxyBuilderExtensions.cs | 8 ++ ...ReverseProxyServiceCollectionExtensions.cs | 1 + src/ReverseProxy/EventIds.cs | 1 + .../Middleware/LoadBalancingMiddleware.cs | 43 ++++++++- .../Management/ReverseProxyConfigManager.cs | 6 +- src/ReverseProxy/Service/Proxy/HttpProxy.cs | 7 +- .../Service/RuntimeModel/BackendConfig.cs | 23 ++++- .../SessionAffinity/AffinitizationResult.cs | 23 +++++ .../ISessionAffinityFeature.cs | 26 +++++ .../ISessionAffinityProvider.cs | 39 ++++++++ .../SessionAffinity/SessionAffinityFeature.cs | 19 ++++ .../SessionAffinityProvider.cs | 95 +++++++++++++++++++ .../DestinationInitializerMiddlewareTests.cs | 3 +- .../Middleware/LoadBalancerMiddlewareTests.cs | 2 +- .../HealthProbe/BackendProberFactoryTests.cs | 3 +- .../Service/HealthProbe/BackendProberTests.cs | 3 +- .../HealthProbe/HealthProbeWorkerTests.cs | 12 ++- .../Service/RuntimeModel/BackendInfoTests.cs | 6 +- 21 files changed, 355 insertions(+), 14 deletions(-) create mode 100644 src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityMode.cs create mode 100644 src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityOptions.cs create mode 100644 src/ReverseProxy/Service/SessionAffinity/AffinitizationResult.cs create mode 100644 src/ReverseProxy/Service/SessionAffinity/ISessionAffinityFeature.cs create mode 100644 src/ReverseProxy/Service/SessionAffinity/ISessionAffinityProvider.cs create mode 100644 src/ReverseProxy/Service/SessionAffinity/SessionAffinityFeature.cs create mode 100644 src/ReverseProxy/Service/SessionAffinity/SessionAffinityProvider.cs diff --git a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/Backend.cs b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/Backend.cs index b2cf47d74..a74b59ef2 100644 --- a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/Backend.cs +++ b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/Backend.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract; namespace Microsoft.ReverseProxy.Abstractions { @@ -39,6 +40,11 @@ public sealed class Backend : IDeepCloneable /// public LoadBalancingOptions LoadBalancing { get; set; } + /// + /// Session affinity options. + /// + public SessionAffinityOptions SessionAffinity { get; set; } + /// /// Active health checking options. /// @@ -64,6 +70,7 @@ Backend IDeepCloneable.DeepClone() QuotaOptions = QuotaOptions?.DeepClone(), PartitioningOptions = PartitioningOptions?.DeepClone(), LoadBalancing = LoadBalancing?.DeepClone(), + SessionAffinity = SessionAffinity?.DeepClone(), HealthCheckOptions = HealthCheckOptions?.DeepClone(), Destinations = Destinations.DeepClone(StringComparer.Ordinal), Metadata = Metadata?.DeepClone(StringComparer.Ordinal), diff --git a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityMode.cs b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityMode.cs new file mode 100644 index 000000000..03060e52c --- /dev/null +++ b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityMode.cs @@ -0,0 +1,18 @@ +namespace Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract +{ + /// + /// Location of a session key for affinitized requests + /// + public enum SessionAffinityMode + { + /// + /// Session key is stored as a cookie. + /// + Cookie, + + /// + /// Session key is stored on a custom header. + /// + CustomHeader + } +} diff --git a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityOptions.cs b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityOptions.cs new file mode 100644 index 000000000..7322ff7ec --- /dev/null +++ b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityOptions.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract +{ + /// + /// Session affinitity options. + /// + public sealed class SessionAffinityOptions + { + public SessionAffinityMode Mode { get; set; } + + public string CustomHeaderName { get; set; } + + internal SessionAffinityOptions DeepClone() + { + return new SessionAffinityOptions + { + Mode = Mode, + CustomHeaderName = CustomHeaderName + }; + } + } +} diff --git a/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs b/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs index 8f20ddfa6..48eeab035 100644 --- a/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs +++ b/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs @@ -10,6 +10,7 @@ using Microsoft.ReverseProxy.Service.Metrics; using Microsoft.ReverseProxy.Service.Proxy; using Microsoft.ReverseProxy.Service.Proxy.Infrastructure; +using Microsoft.ReverseProxy.Service.SessionAffinity; using Microsoft.ReverseProxy.Telemetry; using Microsoft.ReverseProxy.Utilities; @@ -82,5 +83,12 @@ public static IReverseProxyBuilder AddBackgroundWorkers(this IReverseProxyBuilde return builder; } + + public static IReverseProxyBuilder AddSessionAffinityProvider(this IReverseProxyBuilder builder) + { + builder.Services.TryAddSingleton(); + + return builder; + } } } diff --git a/src/ReverseProxy/Configuration/DependencyInjection/ReverseProxyServiceCollectionExtensions.cs b/src/ReverseProxy/Configuration/DependencyInjection/ReverseProxyServiceCollectionExtensions.cs index d45d9b9db..a4462253d 100644 --- a/src/ReverseProxy/Configuration/DependencyInjection/ReverseProxyServiceCollectionExtensions.cs +++ b/src/ReverseProxy/Configuration/DependencyInjection/ReverseProxyServiceCollectionExtensions.cs @@ -29,6 +29,7 @@ public static IReverseProxyBuilder AddReverseProxy(this IServiceCollection servi .AddRuntimeStateManagers() .AddConfigManager() .AddDynamicEndpointDataSource() + .AddSessionAffinityProvider() .AddProxy() .AddBackgroundWorkers(); diff --git a/src/ReverseProxy/EventIds.cs b/src/ReverseProxy/EventIds.cs index 683177ffc..f4f2d9d04 100644 --- a/src/ReverseProxy/EventIds.cs +++ b/src/ReverseProxy/EventIds.cs @@ -40,5 +40,6 @@ internal static class EventIds public static readonly EventId OperationStarted = new EventId(31, "OperationStarted"); public static readonly EventId OperationEnded = new EventId(32, "OperationEnded"); public static readonly EventId OperationFailed = new EventId(33, "OperationFailed"); + public static readonly EventId AffinitizedDestinationIsNotFound = new EventId(34, "AffinitizedDestinationIsNotFound"); } } diff --git a/src/ReverseProxy/Middleware/LoadBalancingMiddleware.cs b/src/ReverseProxy/Middleware/LoadBalancingMiddleware.cs index b114671f7..6b1f53ad4 100644 --- a/src/ReverseProxy/Middleware/LoadBalancingMiddleware.cs +++ b/src/ReverseProxy/Middleware/LoadBalancingMiddleware.cs @@ -8,6 +8,7 @@ using Microsoft.ReverseProxy.Abstractions.Telemetry; using Microsoft.ReverseProxy.RuntimeModel; using Microsoft.ReverseProxy.Service.Proxy; +using Microsoft.ReverseProxy.Service.SessionAffinity; namespace Microsoft.ReverseProxy.Middleware { @@ -19,18 +20,21 @@ internal class LoadBalancingMiddleware private readonly ILogger _logger; private readonly IOperationLogger _operationLogger; private readonly ILoadBalancer _loadBalancer; + private readonly ISessionAffinityProvider _sessionAffinityProvider; private readonly RequestDelegate _next; public LoadBalancingMiddleware( RequestDelegate next, ILogger logger, IOperationLogger operationLogger, - ILoadBalancer loadBalancer) + ILoadBalancer loadBalancer, + ISessionAffinityProvider sessionAffinityProvider) { _next = next ?? throw new ArgumentNullException(nameof(next)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _operationLogger = operationLogger ?? throw new ArgumentNullException(nameof(operationLogger)); _loadBalancer = loadBalancer ?? throw new ArgumentNullException(nameof(loadBalancer)); + _sessionAffinityProvider = sessionAffinityProvider ?? throw new ArgumentNullException(nameof(sessionAffinityProvider)); } public Task Invoke(HttpContext context) @@ -42,6 +46,28 @@ public Task Invoke(HttpContext context) var loadBalancingOptions = backend.Config.Value?.LoadBalancingOptions ?? new BackendConfig.BackendLoadBalancingOptions(default); + var sessionAffinityOptions = backend.Config.Value?.SessionAffinityOptions + ?? new BackendConfig.BackendSessionAffinityOptions(false, default, default); + + var isAffinitized = false; + if (sessionAffinityOptions.Enabled) + { + var affinitizedDestinations = _sessionAffinityProvider.TryFindAffinitizedDestinations(context, destinations, sessionAffinityOptions); + if (affinitizedDestinations.RequestKeyFound) + { + isAffinitized = true; + if (affinitizedDestinations.Destinations.Count > 0) + { + destinations = affinitizedDestinations.Destinations; + } + else + { + Log.AffinitizedDestinationIsNotFound(_logger, affinitizedDestinations.RequestKey, backend.BackendId); + context.Response.StatusCode = 503; + return Task.CompletedTask; + } + } + } var destination = _operationLogger.Execute( "ReverseProxy.PickDestination", @@ -54,6 +80,11 @@ public Task Invoke(HttpContext context) return Task.CompletedTask; } + if(sessionAffinityOptions.Enabled && !isAffinitized) + { + _sessionAffinityProvider.AffinitizeRequest(context, sessionAffinityOptions, destination); + } + destinationsFeature.Destinations = new[] { destination }; return _next(context); @@ -66,10 +97,20 @@ private static class Log EventIds.NoAvailableDestinations, "No available destinations after load balancing for backend `{backendId}`."); + private static readonly Action _affinitizedDestinationIsNotFound = LoggerMessage.Define( + LogLevel.Warning, + EventIds.AffinitizedDestinationIsNotFound, + "No destinations found for the affinitized request with key `{affinityKey}` on backend `{backendId}`."); + public static void NoAvailableDestinations(ILogger logger, string backendId) { _noAvailableDestinations(logger, backendId, null); } + + public static void AffinitizedDestinationIsNotFound(ILogger logger, string affinityKey, string backendId) + { + _affinitizedDestinationIsNotFound(logger, affinityKey, backendId, null); + } } } } diff --git a/src/ReverseProxy/Service/Management/ReverseProxyConfigManager.cs b/src/ReverseProxy/Service/Management/ReverseProxyConfigManager.cs index 1ae6cd0cc..e6a9f9805 100644 --- a/src/ReverseProxy/Service/Management/ReverseProxyConfigManager.cs +++ b/src/ReverseProxy/Service/Management/ReverseProxyConfigManager.cs @@ -94,7 +94,11 @@ private void UpdateRuntimeBackends(DynamicConfigRoot config) port: configBackend.HealthCheckOptions?.Port ?? 0, path: configBackend.HealthCheckOptions?.Path ?? string.Empty), new BackendConfig.BackendLoadBalancingOptions( - mode: configBackend.LoadBalancing?.Mode ?? default)); + mode: configBackend.LoadBalancing?.Mode ?? default), + new BackendConfig.BackendSessionAffinityOptions( + enabled: configBackend.SessionAffinity != null, + mode: configBackend.SessionAffinity?.Mode ?? default, + customHeaderName: configBackend.SessionAffinity?.CustomHeaderName)); var currentBackendConfig = backend.Config.Value; if (currentBackendConfig == null || diff --git a/src/ReverseProxy/Service/Proxy/HttpProxy.cs b/src/ReverseProxy/Service/Proxy/HttpProxy.cs index 5fc16898f..79c7a8f3a 100644 --- a/src/ReverseProxy/Service/Proxy/HttpProxy.cs +++ b/src/ReverseProxy/Service/Proxy/HttpProxy.cs @@ -19,6 +19,7 @@ using Microsoft.Extensions.Primitives; using Microsoft.ReverseProxy.Service.Metrics; using Microsoft.ReverseProxy.Service.Proxy.Infrastructure; +using Microsoft.ReverseProxy.Service.SessionAffinity; using Microsoft.ReverseProxy.Utilities; namespace Microsoft.ReverseProxy.Service.Proxy @@ -42,13 +43,16 @@ internal class HttpProxy : IHttpProxy private readonly ILogger _logger; private readonly ProxyMetrics _metrics; + private readonly ISessionAffinityProvider _sessionAffinityProvider; - public HttpProxy(ILogger logger, ProxyMetrics metrics) + public HttpProxy(ILogger logger, ProxyMetrics metrics, ISessionAffinityProvider sessionAffinityProvider) { Contracts.CheckValue(logger, nameof(logger)); Contracts.CheckValue(metrics, nameof(metrics)); + Contracts.CheckValue(sessionAffinityProvider, nameof(sessionAffinityProvider)); _logger = logger; _metrics = metrics; + _sessionAffinityProvider = sessionAffinityProvider; } /// @@ -296,6 +300,7 @@ private async Task UpgradableProxyAsync( // ::::::::::::::::::::::::::::::::::::::::::::: // :: Step 5: Copy response headers Downstream ◄-- Proxy ◄-- Upstream CopyHeadersToDownstream(upstreamResponse, context.Response.Headers); + _sessionAffinityProvider.SetAffinityKeyOnDownstreamResponse(context); if (!upgraded) { diff --git a/src/ReverseProxy/Service/RuntimeModel/BackendConfig.cs b/src/ReverseProxy/Service/RuntimeModel/BackendConfig.cs index 387f4c900..9e1ed0324 100644 --- a/src/ReverseProxy/Service/RuntimeModel/BackendConfig.cs +++ b/src/ReverseProxy/Service/RuntimeModel/BackendConfig.cs @@ -3,6 +3,7 @@ using System; using Microsoft.ReverseProxy.Abstractions; +using Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract; using Microsoft.ReverseProxy.Utilities; namespace Microsoft.ReverseProxy.RuntimeModel @@ -21,16 +22,20 @@ internal sealed class BackendConfig { public BackendConfig( BackendHealthCheckOptions healthCheckOptions, - BackendLoadBalancingOptions loadBalancingOptions) + BackendLoadBalancingOptions loadBalancingOptions, + BackendSessionAffinityOptions sessionAffinityOptions) { HealthCheckOptions = healthCheckOptions; LoadBalancingOptions = loadBalancingOptions; + SessionAffinityOptions = sessionAffinityOptions; } public BackendHealthCheckOptions HealthCheckOptions { get; } public BackendLoadBalancingOptions LoadBalancingOptions { get; } + public BackendSessionAffinityOptions SessionAffinityOptions { get; } + /// /// Active health probing options for a backend. /// @@ -88,5 +93,21 @@ public BackendLoadBalancingOptions(LoadBalancingMode mode) internal AtomicCounter RoundRobinState { get; } } + + internal readonly struct BackendSessionAffinityOptions + { + public BackendSessionAffinityOptions(bool enabled, SessionAffinityMode mode, string customHeaderName) + { + Mode = mode; + CustomHeaderName = customHeaderName; + Enabled = enabled; + } + + public bool Enabled { get; } + + public SessionAffinityMode Mode { get; } + + public string CustomHeaderName { get; } + } } } diff --git a/src/ReverseProxy/Service/SessionAffinity/AffinitizationResult.cs b/src/ReverseProxy/Service/SessionAffinity/AffinitizationResult.cs new file mode 100644 index 000000000..2dbd1aef6 --- /dev/null +++ b/src/ReverseProxy/Service/SessionAffinity/AffinitizationResult.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Collections.Generic; +using Microsoft.ReverseProxy.RuntimeModel; + +namespace Microsoft.ReverseProxy.Service.SessionAffinity +{ + public readonly struct AffinitizedDestinationCollection + { + public readonly IReadOnlyList Destinations; + + public readonly string RequestKey; + + public AffinitizedDestinationCollection(IReadOnlyList destinations, string requestKey) + { + Destinations = destinations; + RequestKey = requestKey; + } + + public bool RequestKeyFound => RequestKey != null; + } +} diff --git a/src/ReverseProxy/Service/SessionAffinity/ISessionAffinityFeature.cs b/src/ReverseProxy/Service/SessionAffinity/ISessionAffinityFeature.cs new file mode 100644 index 000000000..7a8c36046 --- /dev/null +++ b/src/ReverseProxy/Service/SessionAffinity/ISessionAffinityFeature.cs @@ -0,0 +1,26 @@ +using Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract; +using Microsoft.ReverseProxy.RuntimeModel; + +namespace Microsoft.ReverseProxy.Service.SessionAffinity +{ + /// + /// Tracks current the request's affinity. + /// + public interface ISessionAffinityFeature + { + /// + /// Key binding the current request to one or many . + /// + public string DestinationKey { get; set; } + + /// + /// Affinity mode. + /// + public SessionAffinityMode Mode { get; set; } + + /// + /// Name of a custom header storing an affinity key to be used when is set to . + /// + public string CustomHeaderName { get; set; } + } +} diff --git a/src/ReverseProxy/Service/SessionAffinity/ISessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/ISessionAffinityProvider.cs new file mode 100644 index 000000000..1efd5dd81 --- /dev/null +++ b/src/ReverseProxy/Service/SessionAffinity/ISessionAffinityProvider.cs @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Collections.Generic; +using System.Net.Http; +using Microsoft.AspNetCore.Http; +using Microsoft.ReverseProxy.RuntimeModel; + +namespace Microsoft.ReverseProxy.Service.SessionAffinity +{ + /// + /// Provides session affinity for load-balanced backends. + /// + internal interface ISessionAffinityProvider + { + /// + /// Tries to find to which the current request is affinitized by the affinity key. + /// + /// Current request's context. + /// s available for the request. + /// Affinity options. + /// . + public AffinitizedDestinationCollection TryFindAffinitizedDestinations(HttpContext context, IReadOnlyList destinations, BackendConfig.BackendSessionAffinityOptions options); + + /// + /// Affinitize the current request to the given by setting the affinity key extracted from . + /// + /// Current request's context. + /// Affinity options. + /// to which request is to be affinitized. + public void AffinitizeRequest(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, DestinationInfo destination); + + /// + /// Sets an affinity key on a downstream response if any is defined by . + /// + /// Request context. + public void SetAffinityKeyOnDownstreamResponse(HttpContext context); + } +} diff --git a/src/ReverseProxy/Service/SessionAffinity/SessionAffinityFeature.cs b/src/ReverseProxy/Service/SessionAffinity/SessionAffinityFeature.cs new file mode 100644 index 000000000..928e70a8b --- /dev/null +++ b/src/ReverseProxy/Service/SessionAffinity/SessionAffinityFeature.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract; + +namespace Microsoft.ReverseProxy.Service.SessionAffinity +{ + internal class SessionAffinityFeature : ISessionAffinityFeature + { + /// + public string DestinationKey { get; set; } + + /// + public SessionAffinityMode Mode { get; set; } + + /// + public string CustomHeaderName { get; set; } + } +} diff --git a/src/ReverseProxy/Service/SessionAffinity/SessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/SessionAffinityProvider.cs new file mode 100644 index 000000000..dfcd94c40 --- /dev/null +++ b/src/ReverseProxy/Service/SessionAffinity/SessionAffinityProvider.cs @@ -0,0 +1,95 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Primitives; +using Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract; +using Microsoft.ReverseProxy.RuntimeModel; + +namespace Microsoft.ReverseProxy.Service.SessionAffinity +{ + internal class SessionAffinityProvider : ISessionAffinityProvider + { + private const string CookieName = "ms-rev-proxy-sess-key"; + + //TBD. Add logging. + + public void AffinitizeRequest(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, DestinationInfo destination) + { + if (!options.Enabled) + { + return; + } + + var affinityKey = destination.DestinationId; // Currently, we always use ID as the affinity key + + context.Features.Set(new SessionAffinityFeature { DestinationKey = affinityKey, Mode = options.Mode, CustomHeaderName = options.CustomHeaderName }); + } + + public AffinitizedDestinationCollection TryFindAffinitizedDestinations(HttpContext context, IReadOnlyList destinations, BackendConfig.BackendSessionAffinityOptions options) + { + if (!options.Enabled || destinations.Count == 0) + { + return default; + } + + var requestAffinityKey = GetAffinityKey(context, options); + + // TBD. Support different failure modes + if (requestAffinityKey == null) + { + return default; + } + + context.Features.Set(new SessionAffinityFeature { DestinationKey = requestAffinityKey, Mode = options.Mode, CustomHeaderName = options.CustomHeaderName }); + + var matchingDestinations = new List(); + for(var i = 0; i < destinations.Count; i++) + { + if (destinations[i].DestinationId == requestAffinityKey) + { + matchingDestinations.Add(destinations[i]); + } + } + + return new AffinitizedDestinationCollection(matchingDestinations, requestAffinityKey); + } + + public void SetAffinityKeyOnDownstreamResponse(HttpContext context) + { + var affinityFeature = context.Features.Get(); + + if (affinityFeature == null) + { + return; + } + + // TBD. The affinity key must be encrypted. + switch (affinityFeature.Mode) + { + case SessionAffinityMode.Cookie: + context.Response.Cookies.Append(CookieName, affinityFeature.DestinationKey); + return; + case SessionAffinityMode.CustomHeader: + context.Response.Headers.Add(affinityFeature.CustomHeaderName, new StringValues(affinityFeature.DestinationKey)); + return; + } + } + + private string GetAffinityKey(HttpContext context, BackendConfig.BackendSessionAffinityOptions options) + { + switch (options.Mode) + { + case SessionAffinityMode.Cookie: + return context.Request.Cookies.TryGetValue(CookieName, out var keyInCookie) ? keyInCookie : null; + case SessionAffinityMode.CustomHeader: + var keyHeaderValues = context.Request.Headers[options.CustomHeaderName]; + return !StringValues.IsNullOrEmpty(keyHeaderValues) ? keyHeaderValues[0] : null; // We always take the first value of a custom header storing an affinity key + default: + throw new ArgumentOutOfRangeException(nameof(options), $"Unsupported value {options.Mode} of {nameof(SessionAffinityMode)}."); + } + } + } +} diff --git a/test/ReverseProxy.Tests/Middleware/DestinationInitializerMiddlewareTests.cs b/test/ReverseProxy.Tests/Middleware/DestinationInitializerMiddlewareTests.cs index 40cef94a7..dc01dbdb5 100644 --- a/test/ReverseProxy.Tests/Middleware/DestinationInitializerMiddlewareTests.cs +++ b/test/ReverseProxy.Tests/Middleware/DestinationInitializerMiddlewareTests.cs @@ -85,7 +85,8 @@ public async Task Invoke_NoHealthyEndpoints_503() proxyHttpClientFactory: proxyHttpClientFactoryMock.Object); backend1.Config.Value = new BackendConfig( new BackendConfig.BackendHealthCheckOptions(enabled: true, Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan, 0, ""), - new BackendConfig.BackendLoadBalancingOptions()); + new BackendConfig.BackendLoadBalancingOptions(), + new BackendConfig.BackendSessionAffinityOptions()); var destination1 = backend1.DestinationManager.GetOrCreateItem( "destination1", destination => diff --git a/test/ReverseProxy.Tests/Middleware/LoadBalancerMiddlewareTests.cs b/test/ReverseProxy.Tests/Middleware/LoadBalancerMiddlewareTests.cs index 1ad19414f..cda5dda5c 100644 --- a/test/ReverseProxy.Tests/Middleware/LoadBalancerMiddlewareTests.cs +++ b/test/ReverseProxy.Tests/Middleware/LoadBalancerMiddlewareTests.cs @@ -41,7 +41,7 @@ public async Task Invoke_Works() backendId: "backend1", destinationManager: new DestinationManager(), proxyHttpClientFactory: proxyHttpClientFactoryMock.Object); - backend1.Config.Value = new BackendConfig(default, new BackendConfig.BackendLoadBalancingOptions(LoadBalancingMode.RoundRobin)); + backend1.Config.Value = new BackendConfig(default, new BackendConfig.BackendLoadBalancingOptions(LoadBalancingMode.RoundRobin), default); var destination1 = backend1.DestinationManager.GetOrCreateItem( "destination1", destination => diff --git a/test/ReverseProxy.Tests/Service/HealthProbe/BackendProberFactoryTests.cs b/test/ReverseProxy.Tests/Service/HealthProbe/BackendProberFactoryTests.cs index 3bd9fa74f..4f476b494 100644 --- a/test/ReverseProxy.Tests/Service/HealthProbe/BackendProberFactoryTests.cs +++ b/test/ReverseProxy.Tests/Service/HealthProbe/BackendProberFactoryTests.cs @@ -54,7 +54,8 @@ public void BackendProberFactory_CreateBackendProber() timeout: TimeSpan.FromSeconds(60), port: 8000, path: "/example"), - loadBalancingOptions: default); + loadBalancingOptions: default, + sessionAffinityOptions: default); var destinationManager = new DestinationManager(); var prober = factory.CreateBackendProber(backendId, backendConfig, destinationManager); diff --git a/test/ReverseProxy.Tests/Service/HealthProbe/BackendProberTests.cs b/test/ReverseProxy.Tests/Service/HealthProbe/BackendProberTests.cs index af4c33ab2..964cc3ed1 100644 --- a/test/ReverseProxy.Tests/Service/HealthProbe/BackendProberTests.cs +++ b/test/ReverseProxy.Tests/Service/HealthProbe/BackendProberTests.cs @@ -52,7 +52,8 @@ public BackendProberTests() timeout: TimeSpan.FromSeconds(60), port: 8000, path: "/example"), - loadBalancingOptions: default); + loadBalancingOptions: default, + sessionAffinityOptions: default); _timer = new VirtualMonotonicTimer(); _semaphore = new AsyncSemaphore(10); _fakeRandom = new Mock(); diff --git a/test/ReverseProxy.Tests/Service/HealthProbe/HealthProbeWorkerTests.cs b/test/ReverseProxy.Tests/Service/HealthProbe/HealthProbeWorkerTests.cs index 5c09c1610..4e9ee1a0b 100644 --- a/test/ReverseProxy.Tests/Service/HealthProbe/HealthProbeWorkerTests.cs +++ b/test/ReverseProxy.Tests/Service/HealthProbe/HealthProbeWorkerTests.cs @@ -41,7 +41,8 @@ public HealthProbeWorkerTests() timeout: TimeSpan.FromSeconds(1), port: 8000, path: "/example"), - loadBalancingOptions: default); + loadBalancingOptions: default, + sessionAffinityOptions: default); // Set up prober. We do not want to let prober really perform any actions. // The behavior of prober should be tested in its own unit test, see "BackendProberTests.cs". @@ -118,7 +119,8 @@ public async Task UpdateTrackedBackends_ProbeNotEnabled_ShouldNotStartProber() timeout: TimeSpan.FromSeconds(20), port: 1234, path: "/"), - loadBalancingOptions: default); + loadBalancingOptions: default, + sessionAffinityOptions: default); }); // Start probing. @@ -196,7 +198,8 @@ public async Task UpdateTrackedBackends_BackendConfigChange_RecreatesProber() timeout: TimeSpan.FromSeconds(1), port: 8000, path: "/newexample"), - loadBalancingOptions: default); + loadBalancingOptions: default, + sessionAffinityOptions: default); await health.UpdateTrackedBackends(); // After the config is updated, the program should discover this change, create a new prober, @@ -228,7 +231,8 @@ public async Task UpdateTrackedBackends_BackendConfigDisabledProbing_StopsProber timeout: TimeSpan.FromSeconds(1), port: 8000, path: "/newexample"), - loadBalancingOptions: default); + loadBalancingOptions: default, + sessionAffinityOptions: default); await health.UpdateTrackedBackends(); // After the config is updated, the program should discover this change, diff --git a/test/ReverseProxy.Tests/Service/RuntimeModel/BackendInfoTests.cs b/test/ReverseProxy.Tests/Service/RuntimeModel/BackendInfoTests.cs index 333f9ddd8..a91cca113 100644 --- a/test/ReverseProxy.Tests/Service/RuntimeModel/BackendInfoTests.cs +++ b/test/ReverseProxy.Tests/Service/RuntimeModel/BackendInfoTests.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using System; +using Microsoft.AspNetCore.Mvc; using Microsoft.ReverseProxy.Service.Management; using Microsoft.ReverseProxy.Service.Proxy.Infrastructure; using Tests.Common; @@ -76,7 +77,7 @@ public void DynamicState_ReactsToBackendConfigChanges() Assert.NotNull(state1); Assert.Empty(state1.AllDestinations); - backend.Config.Value = new BackendConfig(healthCheckOptions: default, loadBalancingOptions: default); + backend.Config.Value = new BackendConfig(healthCheckOptions: default, loadBalancingOptions: default, sessionAffinityOptions: default); Assert.NotSame(state1, backend.DynamicState.Value); Assert.Empty(backend.DynamicState.Value.AllDestinations); } @@ -145,7 +146,8 @@ private static void EnableHealthChecks(BackendInfo backend) timeout: TimeSpan.FromSeconds(30), port: 30000, path: "/"), - loadBalancingOptions: default); + loadBalancingOptions: default, + sessionAffinityOptions: default); } } } From aca8d94db6eceaf87d7779f9cee34a181b3f283d Mon Sep 17 00:00:00 2001 From: Alexander Nikolaev Date: Mon, 18 May 2020 17:50:52 +0200 Subject: [PATCH 02/30] Comment on affinity to a destination pool --- .../Service/SessionAffinity/SessionAffinityProvider.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ReverseProxy/Service/SessionAffinity/SessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/SessionAffinityProvider.cs index dfcd94c40..1ad1c7009 100644 --- a/src/ReverseProxy/Service/SessionAffinity/SessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/SessionAffinityProvider.cs @@ -45,6 +45,7 @@ public AffinitizedDestinationCollection TryFindAffinitizedDestinations(HttpConte context.Features.Set(new SessionAffinityFeature { DestinationKey = requestAffinityKey, Mode = options.Mode, CustomHeaderName = options.CustomHeaderName }); + // It's allowed to affinitize a request to a pool of destinations so as to enable load-balancing among them var matchingDestinations = new List(); for(var i = 0; i < destinations.Count; i++) { From 0ba155473526be6f5a0ac793081184537b4e53de Mon Sep 17 00:00:00 2001 From: Alexander Nikolaev Date: Tue, 19 May 2020 18:07:30 +0200 Subject: [PATCH 03/30] - Session affinity logic separated from load balancing into 2 new middleware types - Extensible name-based affinity provider configuration - Affinity cookie properties can be customized - HttpProxy calls Append to set downstream response headers - Affinity key is set on a downstream response right after request gets affinitized --- samples/ReverseProxy.Sample/Startup.cs | 2 +- .../CookieSessionAffinityProviderOptions.cs | 45 +++++++++ .../Contract/SessionAffinityMode.cs | 18 ---- .../Contract/SessionAffinityOptions.cs | 15 ++- .../IReverseProxyBuilderExtensions.cs | 9 +- ...rseProxyIEndpointRouteBuilderExtensions.cs | 2 +- src/ReverseProxy/EventIds.cs | 1 + .../Middleware/AffinitizeRequestMiddleware.cs | 68 +++++++++++++ .../AffinitizedDestinationLookupMiddleware.cs | 73 ++++++++++++++ .../HttpContextFeaturesExtensions.cs | 27 ++++++ .../Middleware/LoadBalancingMiddleware.cs | 50 +--------- .../ProxyMiddlewareAppBuilderExtensions.cs | 10 ++ .../SessionAffinityMiddlewareHelper.cs | 41 ++++++++ .../Management/ReverseProxyConfigManager.cs | 2 +- src/ReverseProxy/Service/Proxy/HttpProxy.cs | 8 +- .../Service/RuntimeModel/BackendConfig.cs | 9 +- ...cs => AffinitizedDestinationCollection.cs} | 6 +- .../BaseSessionAffinityProvider.cs | 82 ++++++++++++++++ .../CookieSessionAffinityProvider.cs | 41 ++++++++ .../CustomHeaderSessionAffinityProvider.cs | 37 +++++++ .../ISessionAffinityFeature.cs | 26 ----- .../ISessionAffinityProvider.cs | 16 ++-- .../SessionAffinity/SessionAffinityFeature.cs | 19 ---- .../SessionAffinityProvider.cs | 96 ------------------- 24 files changed, 469 insertions(+), 234 deletions(-) create mode 100644 src/ReverseProxy/Abstractions/BackendDiscovery/Contract/CookieSessionAffinityProviderOptions.cs delete mode 100644 src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityMode.cs create mode 100644 src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs create mode 100644 src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs create mode 100644 src/ReverseProxy/Middleware/HttpContextFeaturesExtensions.cs create mode 100644 src/ReverseProxy/Middleware/SessionAffinityMiddlewareHelper.cs rename src/ReverseProxy/Service/SessionAffinity/{AffinitizationResult.cs => AffinitizedDestinationCollection.cs} (77%) create mode 100644 src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs create mode 100644 src/ReverseProxy/Service/SessionAffinity/CookieSessionAffinityProvider.cs create mode 100644 src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs delete mode 100644 src/ReverseProxy/Service/SessionAffinity/ISessionAffinityFeature.cs delete mode 100644 src/ReverseProxy/Service/SessionAffinity/SessionAffinityFeature.cs delete mode 100644 src/ReverseProxy/Service/SessionAffinity/SessionAffinityProvider.cs diff --git a/samples/ReverseProxy.Sample/Startup.cs b/samples/ReverseProxy.Sample/Startup.cs index a74a197b9..4bbebf768 100644 --- a/samples/ReverseProxy.Sample/Startup.cs +++ b/samples/ReverseProxy.Sample/Startup.cs @@ -62,7 +62,7 @@ public void Configure(IApplicationBuilder app) return next(); }); - proxyPipeline.UseProxyLoadBalancing(); + proxyPipeline.UseProxyLoadBalancingWithSessionAffinity(); }); }); } diff --git a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/CookieSessionAffinityProviderOptions.cs b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/CookieSessionAffinityProviderOptions.cs new file mode 100644 index 000000000..be82f7999 --- /dev/null +++ b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/CookieSessionAffinityProviderOptions.cs @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using Microsoft.AspNetCore.Http; + +namespace Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract +{ + /// + /// Defines cookie-specific affinity provider options. + /// + public class CookieSessionAffinityProviderOptions + { + private CookieBuilder _cookieBuilder = new AffinityCookieBuilder(); + + public static readonly string DefaultCookieName = ".Microsoft.ReverseProxy.Affinity"; + + public static readonly string DefaultCookiePath = "/"; + + public CookieBuilder Cookie + { + get => _cookieBuilder; + set => _cookieBuilder = value ?? throw new ArgumentNullException(nameof(value)); + } + + private class AffinityCookieBuilder : CookieBuilder + { + public AffinityCookieBuilder() + { + Name = DefaultCookieName; + Path = DefaultCookiePath; + SecurePolicy = CookieSecurePolicy.None; + SameSite = SameSiteMode.Lax; + HttpOnly = true; + IsEssential = false; + } + + public override TimeSpan? Expiration + { + get => null; + set => throw new InvalidOperationException(nameof(Expiration) + " cannot be set for the cookie defined by " + nameof(CookieSessionAffinityProviderOptions)); + } + } + } +} diff --git a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityMode.cs b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityMode.cs deleted file mode 100644 index 03060e52c..000000000 --- a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityMode.cs +++ /dev/null @@ -1,18 +0,0 @@ -namespace Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract -{ - /// - /// Location of a session key for affinitized requests - /// - public enum SessionAffinityMode - { - /// - /// Session key is stored as a cookie. - /// - Cookie, - - /// - /// Session key is stored on a custom header. - /// - CustomHeader - } -} diff --git a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityOptions.cs b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityOptions.cs index 7322ff7ec..d7d1d151f 100644 --- a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityOptions.cs +++ b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityOptions.cs @@ -1,6 +1,9 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System; +using System.Collections.Generic; + namespace Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract { /// @@ -8,16 +11,22 @@ namespace Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract /// public sealed class SessionAffinityOptions { - public SessionAffinityMode Mode { get; set; } + /// + /// Session affinity mode which is implemented by one of providers. + /// + public string Mode { get; set; } - public string CustomHeaderName { get; set; } + /// + /// Key-value pair collection holding extra settings specific to different affinity modes. + /// + public IDictionary Settings { get; set; } internal SessionAffinityOptions DeepClone() { return new SessionAffinityOptions { Mode = Mode, - CustomHeaderName = CustomHeaderName + Settings = Settings?.DeepClone(StringComparer.Ordinal) }; } } diff --git a/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs b/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs index 48eeab035..d131bbc13 100644 --- a/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs +++ b/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs @@ -1,8 +1,11 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.ReverseProxy.Abstractions; +using Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract; using Microsoft.ReverseProxy.Abstractions.Telemetry; using Microsoft.ReverseProxy.Abstractions.Time; using Microsoft.ReverseProxy.Service; @@ -86,7 +89,11 @@ public static IReverseProxyBuilder AddBackgroundWorkers(this IReverseProxyBuilde public static IReverseProxyBuilder AddSessionAffinityProvider(this IReverseProxyBuilder builder) { - builder.Services.TryAddSingleton(); + builder.Services.TryAddSingleton(); + builder.Services.TryAddEnumerable(new[] { + new ServiceDescriptor(typeof(ISessionAffinityProvider), typeof(CookieSessionAffinityProvider), ServiceLifetime.Singleton), + new ServiceDescriptor(typeof(ISessionAffinityProvider), typeof(CustomHeaderSessionAffinityProvider), ServiceLifetime.Singleton) + }); return builder; } diff --git a/src/ReverseProxy/Configuration/ReverseProxyIEndpointRouteBuilderExtensions.cs b/src/ReverseProxy/Configuration/ReverseProxyIEndpointRouteBuilderExtensions.cs index beffa6a6d..f486e6ac9 100644 --- a/src/ReverseProxy/Configuration/ReverseProxyIEndpointRouteBuilderExtensions.cs +++ b/src/ReverseProxy/Configuration/ReverseProxyIEndpointRouteBuilderExtensions.cs @@ -22,7 +22,7 @@ public static void MapReverseProxy(this IEndpointRouteBuilder endpoints) { endpoints.MapReverseProxy(app => { - app.UseProxyLoadBalancing(); + app.UseProxyLoadBalancingWithSessionAffinity(); }); } diff --git a/src/ReverseProxy/EventIds.cs b/src/ReverseProxy/EventIds.cs index f4f2d9d04..49bab7ca8 100644 --- a/src/ReverseProxy/EventIds.cs +++ b/src/ReverseProxy/EventIds.cs @@ -41,5 +41,6 @@ internal static class EventIds public static readonly EventId OperationEnded = new EventId(32, "OperationEnded"); public static readonly EventId OperationFailed = new EventId(33, "OperationFailed"); public static readonly EventId AffinitizedDestinationIsNotFound = new EventId(34, "AffinitizedDestinationIsNotFound"); + public static readonly EventId AttemptToAffinitizeMultipleDestinations = new EventId(35, "AttemptToAffinitizeMultipleDestinations"); } } diff --git a/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs b/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs new file mode 100644 index 000000000..8108b1b45 --- /dev/null +++ b/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using Microsoft.ReverseProxy.RuntimeModel; +using Microsoft.ReverseProxy.Service.SessionAffinity; + +namespace Microsoft.ReverseProxy.Middleware +{ + /// + /// Affinitizes request to a chosen . + /// + internal class AffinitizeRequestMiddleware + { + private readonly RequestDelegate _next; + private readonly IDictionary _sessionAffinityProviders = new Dictionary(); + private readonly ILogger _logger; + + public AffinitizeRequestMiddleware(RequestDelegate next, IEnumerable sessionAffinityProviders, ILogger logger) + { + _next = next ?? throw new ArgumentNullException(nameof(next)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _sessionAffinityProviders = sessionAffinityProviders.ToProviderDictionary(); + } + + public Task Invoke(HttpContext context) + { + var backend = context.GetRequiredBacked(); + var options = backend.Config.Value?.SessionAffinityOptions + ?? new BackendConfig.BackendSessionAffinityOptions(false, default, default); + + if (options.Enabled) + { + var destinationsFeature = context.GetRequiredDestinationFeature(); + + if (destinationsFeature.Destinations.Count > 1) + { + Log.AttemptToAffinitizeMultipleDestinations(_logger, backend.BackendId); + } + + var destination = destinationsFeature.Destinations[0]; // We always pick the first destination even if there are still a number of them. + // It's assumed that all of them match to the request's affinity key. + + var currentProvider = _sessionAffinityProviders.GetRequiredProvider(options.Mode); + currentProvider.AffinitizeRequest(context, options, destination); + } + + return _next(context); + } + + private static class Log + { + private static readonly Action _attemptToAffinitizeMultipleDestinations = LoggerMessage.Define( + LogLevel.Warning, + EventIds.AttemptToAffinitizeMultipleDestinations, + "Attempt to affinitize multiple destinations to the same request on backend `{backendId}`. The first destination will be used."); + + public static void AttemptToAffinitizeMultipleDestinations(ILogger logger, string backendId) + { + _attemptToAffinitizeMultipleDestinations(logger, backendId, null); + } + } + } +} diff --git a/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs b/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs new file mode 100644 index 000000000..c25822a93 --- /dev/null +++ b/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using Microsoft.ReverseProxy.RuntimeModel; +using Microsoft.ReverseProxy.Service.SessionAffinity; + +namespace Microsoft.ReverseProxy.Middleware +{ + /// + /// Looks up an affinitized matching the request's affinity key if any is set + /// + internal class AffinitizedDestinationLookupMiddleware + { + private readonly RequestDelegate _next; + private readonly IDictionary _sessionAffinityProviders = new Dictionary(); + private readonly ILogger _logger; + + public AffinitizedDestinationLookupMiddleware(RequestDelegate next, IEnumerable sessionAffinityProviders, ILogger logger) + { + _next = next ?? throw new ArgumentNullException(nameof(next)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _sessionAffinityProviders = sessionAffinityProviders.ToProviderDictionary(); + } + + public Task Invoke(HttpContext context) + { + var backend = context.GetRequiredBacked(); + var destinationsFeature = context.GetRequiredDestinationFeature(); + var destinations = destinationsFeature.Destinations; + + var options = backend.Config.Value?.SessionAffinityOptions + ?? new BackendConfig.BackendSessionAffinityOptions(false, default, default); + + if (options.Enabled) + { + var currentProvider = _sessionAffinityProviders.GetRequiredProvider(options.Mode); + if (currentProvider.TryFindAffinitizedDestinations(context, destinations, options, out var affinitizedDestinations)) + { + if (affinitizedDestinations.Destinations.Count > 0) + { + destinations = affinitizedDestinations.Destinations; + } + else + { + Log.AffinitizedDestinationIsNotFound(_logger, affinitizedDestinations.RequestKey?.ToString(), backend.BackendId); + context.Response.StatusCode = 503; + return Task.CompletedTask; + } + } + } + + return _next(context); + } + + private static class Log + { + private static readonly Action _affinitizedDestinationIsNotFound = LoggerMessage.Define( + LogLevel.Warning, + EventIds.AffinitizedDestinationIsNotFound, + "No destinations found for the affinitized request with key `{affinityKey}` on backend `{backendId}`."); + + public static void AffinitizedDestinationIsNotFound(ILogger logger, string affinityKey, string backendId) + { + _affinitizedDestinationIsNotFound(logger, affinityKey, backendId, null); + } + } + } +} diff --git a/src/ReverseProxy/Middleware/HttpContextFeaturesExtensions.cs b/src/ReverseProxy/Middleware/HttpContextFeaturesExtensions.cs new file mode 100644 index 000000000..129d62654 --- /dev/null +++ b/src/ReverseProxy/Middleware/HttpContextFeaturesExtensions.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using Microsoft.AspNetCore.Http; +using Microsoft.ReverseProxy.RuntimeModel; + +namespace Microsoft.ReverseProxy.Middleware +{ + internal static class HttpContextFeaturesExtensions + { + public static BackendInfo GetRequiredBacked(this HttpContext context) + { + return context.Features.Get() ?? throw new InvalidOperationException("Backend unspecified."); + } + + public static IAvailableDestinationsFeature GetRequiredDestinationFeature(this HttpContext context) + { + var destinationsFeature = context.Features.Get(); + if (destinationsFeature?.Destinations == null) + { + throw new InvalidOperationException("The IAvailableDestinationsFeature Destinations collection was not set."); + } + return destinationsFeature; + } + } +} diff --git a/src/ReverseProxy/Middleware/LoadBalancingMiddleware.cs b/src/ReverseProxy/Middleware/LoadBalancingMiddleware.cs index 6b1f53ad4..94466b575 100644 --- a/src/ReverseProxy/Middleware/LoadBalancingMiddleware.cs +++ b/src/ReverseProxy/Middleware/LoadBalancingMiddleware.cs @@ -8,7 +8,6 @@ using Microsoft.ReverseProxy.Abstractions.Telemetry; using Microsoft.ReverseProxy.RuntimeModel; using Microsoft.ReverseProxy.Service.Proxy; -using Microsoft.ReverseProxy.Service.SessionAffinity; namespace Microsoft.ReverseProxy.Middleware { @@ -20,54 +19,28 @@ internal class LoadBalancingMiddleware private readonly ILogger _logger; private readonly IOperationLogger _operationLogger; private readonly ILoadBalancer _loadBalancer; - private readonly ISessionAffinityProvider _sessionAffinityProvider; private readonly RequestDelegate _next; public LoadBalancingMiddleware( RequestDelegate next, ILogger logger, IOperationLogger operationLogger, - ILoadBalancer loadBalancer, - ISessionAffinityProvider sessionAffinityProvider) + ILoadBalancer loadBalancer) { _next = next ?? throw new ArgumentNullException(nameof(next)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _operationLogger = operationLogger ?? throw new ArgumentNullException(nameof(operationLogger)); _loadBalancer = loadBalancer ?? throw new ArgumentNullException(nameof(loadBalancer)); - _sessionAffinityProvider = sessionAffinityProvider ?? throw new ArgumentNullException(nameof(sessionAffinityProvider)); } public Task Invoke(HttpContext context) { - var backend = context.Features.Get() ?? throw new InvalidOperationException("Backend unspecified."); - var destinationsFeature = context.Features.Get(); - var destinations = destinationsFeature?.Destinations - ?? throw new InvalidOperationException("The IAvailableDestinationsFeature Destinations collection was not set."); + var backend = context.GetRequiredBacked(); + var destinationsFeature = context.GetRequiredDestinationFeature(); + var destinations = destinationsFeature.Destinations; var loadBalancingOptions = backend.Config.Value?.LoadBalancingOptions ?? new BackendConfig.BackendLoadBalancingOptions(default); - var sessionAffinityOptions = backend.Config.Value?.SessionAffinityOptions - ?? new BackendConfig.BackendSessionAffinityOptions(false, default, default); - - var isAffinitized = false; - if (sessionAffinityOptions.Enabled) - { - var affinitizedDestinations = _sessionAffinityProvider.TryFindAffinitizedDestinations(context, destinations, sessionAffinityOptions); - if (affinitizedDestinations.RequestKeyFound) - { - isAffinitized = true; - if (affinitizedDestinations.Destinations.Count > 0) - { - destinations = affinitizedDestinations.Destinations; - } - else - { - Log.AffinitizedDestinationIsNotFound(_logger, affinitizedDestinations.RequestKey, backend.BackendId); - context.Response.StatusCode = 503; - return Task.CompletedTask; - } - } - } var destination = _operationLogger.Execute( "ReverseProxy.PickDestination", @@ -80,11 +53,6 @@ public Task Invoke(HttpContext context) return Task.CompletedTask; } - if(sessionAffinityOptions.Enabled && !isAffinitized) - { - _sessionAffinityProvider.AffinitizeRequest(context, sessionAffinityOptions, destination); - } - destinationsFeature.Destinations = new[] { destination }; return _next(context); @@ -97,20 +65,10 @@ private static class Log EventIds.NoAvailableDestinations, "No available destinations after load balancing for backend `{backendId}`."); - private static readonly Action _affinitizedDestinationIsNotFound = LoggerMessage.Define( - LogLevel.Warning, - EventIds.AffinitizedDestinationIsNotFound, - "No destinations found for the affinitized request with key `{affinityKey}` on backend `{backendId}`."); - public static void NoAvailableDestinations(ILogger logger, string backendId) { _noAvailableDestinations(logger, backendId, null); } - - public static void AffinitizedDestinationIsNotFound(ILogger logger, string affinityKey, string backendId) - { - _affinitizedDestinationIsNotFound(logger, affinityKey, backendId, null); - } } } } diff --git a/src/ReverseProxy/Middleware/ProxyMiddlewareAppBuilderExtensions.cs b/src/ReverseProxy/Middleware/ProxyMiddlewareAppBuilderExtensions.cs index 7fbadf5fe..9095db82b 100644 --- a/src/ReverseProxy/Middleware/ProxyMiddlewareAppBuilderExtensions.cs +++ b/src/ReverseProxy/Middleware/ProxyMiddlewareAppBuilderExtensions.cs @@ -17,5 +17,15 @@ public static IApplicationBuilder UseProxyLoadBalancing(this IApplicationBuilder { return builder.UseMiddleware(); } + + /// + /// Load balances across the available endpoints and maintains session affinity. + /// + public static IApplicationBuilder UseProxyLoadBalancingWithSessionAffinity(this IApplicationBuilder builder) + { + return builder.UseMiddleware() + .UseMiddleware() + .UseMiddleware(); + } } } diff --git a/src/ReverseProxy/Middleware/SessionAffinityMiddlewareHelper.cs b/src/ReverseProxy/Middleware/SessionAffinityMiddlewareHelper.cs new file mode 100644 index 000000000..68bad18fe --- /dev/null +++ b/src/ReverseProxy/Middleware/SessionAffinityMiddlewareHelper.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using Microsoft.ReverseProxy.Service.SessionAffinity; + +namespace Microsoft.ReverseProxy.Middleware +{ + internal static class SessionAffinityMiddlewareHelper + { + public static IDictionary ToProviderDictionary(this IEnumerable sessionAffinityProviders) + { + if (sessionAffinityProviders == null) + { + throw new ArgumentNullException(nameof(sessionAffinityProviders)); + } + + var result = new Dictionary(); + + foreach (var provider in sessionAffinityProviders) + { + if (!result.TryAdd(provider.Mode, provider)) + { + throw new ArgumentException(nameof(sessionAffinityProviders), $"More than one {nameof(ISessionAffinityProvider)} found with the same Mode value."); + } + } + + return result; + } + + public static ISessionAffinityProvider GetRequiredProvider(this IDictionary sessionAffinityProviders, string mode) + { + if (!sessionAffinityProviders.TryGetValue(mode, out var currentProvider)) + { + throw new ArgumentException(nameof(mode), $"No {nameof(ISessionAffinityProvider)} was found for the mode {mode}."); + } + return currentProvider; + } + } +} diff --git a/src/ReverseProxy/Service/Management/ReverseProxyConfigManager.cs b/src/ReverseProxy/Service/Management/ReverseProxyConfigManager.cs index e6a9f9805..0884c89ed 100644 --- a/src/ReverseProxy/Service/Management/ReverseProxyConfigManager.cs +++ b/src/ReverseProxy/Service/Management/ReverseProxyConfigManager.cs @@ -98,7 +98,7 @@ private void UpdateRuntimeBackends(DynamicConfigRoot config) new BackendConfig.BackendSessionAffinityOptions( enabled: configBackend.SessionAffinity != null, mode: configBackend.SessionAffinity?.Mode ?? default, - customHeaderName: configBackend.SessionAffinity?.CustomHeaderName)); + settings: configBackend.SessionAffinity?.Settings)); var currentBackendConfig = backend.Config.Value; if (currentBackendConfig == null || diff --git a/src/ReverseProxy/Service/Proxy/HttpProxy.cs b/src/ReverseProxy/Service/Proxy/HttpProxy.cs index 79c7a8f3a..81fc8e545 100644 --- a/src/ReverseProxy/Service/Proxy/HttpProxy.cs +++ b/src/ReverseProxy/Service/Proxy/HttpProxy.cs @@ -43,16 +43,13 @@ internal class HttpProxy : IHttpProxy private readonly ILogger _logger; private readonly ProxyMetrics _metrics; - private readonly ISessionAffinityProvider _sessionAffinityProvider; - public HttpProxy(ILogger logger, ProxyMetrics metrics, ISessionAffinityProvider sessionAffinityProvider) + public HttpProxy(ILogger logger, ProxyMetrics metrics) { Contracts.CheckValue(logger, nameof(logger)); Contracts.CheckValue(metrics, nameof(metrics)); - Contracts.CheckValue(sessionAffinityProvider, nameof(sessionAffinityProvider)); _logger = logger; _metrics = metrics; - _sessionAffinityProvider = sessionAffinityProvider; } /// @@ -300,7 +297,6 @@ private async Task UpgradableProxyAsync( // ::::::::::::::::::::::::::::::::::::::::::::: // :: Step 5: Copy response headers Downstream ◄-- Proxy ◄-- Upstream CopyHeadersToDownstream(upstreamResponse, context.Response.Headers); - _sessionAffinityProvider.SetAffinityKeyOnDownstreamResponse(context); if (!upgraded) { @@ -455,7 +451,7 @@ static void CopyHeaders(HttpHeaders source, IHeaderDictionary destination) } ////this.logger.LogInformation($" Copying downstream <-- Proxy <-- upstream response header {header.Key}: {string.Join(",", header.Value)}"); - destination.TryAdd(header.Key, new StringValues(header.Value.ToArray())); + destination.Append(header.Key, new StringValues(header.Value.ToArray())); } } } diff --git a/src/ReverseProxy/Service/RuntimeModel/BackendConfig.cs b/src/ReverseProxy/Service/RuntimeModel/BackendConfig.cs index 9e1ed0324..075001a00 100644 --- a/src/ReverseProxy/Service/RuntimeModel/BackendConfig.cs +++ b/src/ReverseProxy/Service/RuntimeModel/BackendConfig.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using System; +using System.Collections.Generic; using Microsoft.ReverseProxy.Abstractions; using Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract; using Microsoft.ReverseProxy.Utilities; @@ -96,18 +97,18 @@ public BackendLoadBalancingOptions(LoadBalancingMode mode) internal readonly struct BackendSessionAffinityOptions { - public BackendSessionAffinityOptions(bool enabled, SessionAffinityMode mode, string customHeaderName) + public BackendSessionAffinityOptions(bool enabled, string mode, IDictionary settings) { Mode = mode; - CustomHeaderName = customHeaderName; + Settings = (IReadOnlyDictionary) settings; // Assume that actual dictionary type always implements IReadOnlyDictionary Enabled = enabled; } public bool Enabled { get; } - public SessionAffinityMode Mode { get; } + public string Mode { get; } - public string CustomHeaderName { get; } + public IReadOnlyDictionary Settings { get; } } } } diff --git a/src/ReverseProxy/Service/SessionAffinity/AffinitizationResult.cs b/src/ReverseProxy/Service/SessionAffinity/AffinitizedDestinationCollection.cs similarity index 77% rename from src/ReverseProxy/Service/SessionAffinity/AffinitizationResult.cs rename to src/ReverseProxy/Service/SessionAffinity/AffinitizedDestinationCollection.cs index 2dbd1aef6..f15b71c97 100644 --- a/src/ReverseProxy/Service/SessionAffinity/AffinitizationResult.cs +++ b/src/ReverseProxy/Service/SessionAffinity/AffinitizedDestinationCollection.cs @@ -10,14 +10,12 @@ public readonly struct AffinitizedDestinationCollection { public readonly IReadOnlyList Destinations; - public readonly string RequestKey; + public readonly object RequestKey; - public AffinitizedDestinationCollection(IReadOnlyList destinations, string requestKey) + public AffinitizedDestinationCollection(IReadOnlyList destinations, object requestKey) { Destinations = destinations; RequestKey = requestKey; } - - public bool RequestKeyFound => RequestKey != null; } } diff --git a/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs new file mode 100644 index 000000000..df6c13b8f --- /dev/null +++ b/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using Microsoft.AspNetCore.Http; +using Microsoft.ReverseProxy.RuntimeModel; + +namespace Microsoft.ReverseProxy.Service.SessionAffinity +{ + internal abstract class BaseSessionAffinityProvider : ISessionAffinityProvider + { + protected static readonly object AffinityKeyId = new object(); + + public abstract string Mode { get; } + + public virtual void AffinitizeRequest(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, DestinationInfo destination) + { + if (!options.Enabled) + { + return; + } + + if (!context.Items.TryGetValue(AffinityKeyId, out var affinityKey)) // If affinity key is already set on request, we assume that passed destination always matches to that key + { + affinityKey = GetDestinationAffinityKey(destination); + } + + var encryptedKey = (string)affinityKey; // TBD. The affinity key must be encrypted. + SetEncryptedAffinityKey(context, options, encryptedKey); + } + + public virtual bool TryFindAffinitizedDestinations(HttpContext context, IReadOnlyList destinations, BackendConfig.BackendSessionAffinityOptions options, out AffinitizedDestinationCollection affinitizedDestinations) + { + if (!options.Enabled || destinations.Count == 0) + { + affinitizedDestinations = default; + return false; + } + + var requestAffinityKey = GetRequestAffinityKey(context, options); + + // TBD. Support different failure modes + if (requestAffinityKey == null) + { + affinitizedDestinations = default; + return false; + } + + context.Items.Add(AffinityKeyId, requestAffinityKey); + + // It's allowed to affinitize a request to a pool of destinations so as to enable load-balancing among them + var matchingDestinations = new List(); + for (var i = 0; i < destinations.Count; i++) + { + if (requestAffinityKey.Equals(GetDestinationAffinityKey(destinations[i]))) + { + matchingDestinations.Add(destinations[i]); + } + } + + affinitizedDestinations = new AffinitizedDestinationCollection(matchingDestinations, requestAffinityKey); + return true; + } + + protected virtual string GetSettingValue(string key, BackendConfig.BackendSessionAffinityOptions options) + { + if (options.Settings.TryGetValue(key, out var value)) + { + throw new ArgumentException(nameof(options), $"{nameof(CookieSessionAffinityProvider)} couldn't find the required parameter {key} in session affinity settings."); + } + + return value; + } + + protected abstract T GetDestinationAffinityKey(DestinationInfo destination); + + protected abstract T GetRequestAffinityKey(HttpContext context, BackendConfig.BackendSessionAffinityOptions options); + + protected abstract void SetEncryptedAffinityKey(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, string encryptedKey); + } +} diff --git a/src/ReverseProxy/Service/SessionAffinity/CookieSessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/CookieSessionAffinityProvider.cs new file mode 100644 index 000000000..407269fbb --- /dev/null +++ b/src/ReverseProxy/Service/SessionAffinity/CookieSessionAffinityProvider.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using Microsoft.AspNetCore.Http; +using Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract; +using Microsoft.ReverseProxy.RuntimeModel; + +namespace Microsoft.ReverseProxy.Service.SessionAffinity +{ + internal class CookieSessionAffinityProvider : BaseSessionAffinityProvider + { + private readonly CookieSessionAffinityProviderOptions _providerOptions; + + public CookieSessionAffinityProvider(CookieSessionAffinityProviderOptions providerOptions) + { + _providerOptions = providerOptions ?? throw new ArgumentNullException(nameof(providerOptions)); + } + + public override string Mode => "Cookie"; + + //TBD. Add logging. + + protected override string GetDestinationAffinityKey(DestinationInfo destination) + { + return destination.DestinationId; + } + + protected override string GetRequestAffinityKey(HttpContext context, BackendConfig.BackendSessionAffinityOptions options) + { + return context.Request.Cookies.TryGetValue(_providerOptions.Cookie.Name, out var keyInCookie) ? keyInCookie : null; + } + + protected override void SetEncryptedAffinityKey(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, string encryptedKey) + { + var affinityCookieOptions = _providerOptions.Cookie.Build(context); + // TBD. The affinity key must be encrypted. + context.Response.Cookies.Append(_providerOptions.Cookie.Name, encryptedKey, affinityCookieOptions); + } + } +} diff --git a/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs new file mode 100644 index 000000000..b9d7f5ff1 --- /dev/null +++ b/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Primitives; +using Microsoft.ReverseProxy.RuntimeModel; + +namespace Microsoft.ReverseProxy.Service.SessionAffinity +{ + internal class CustomHeaderSessionAffinityProvider : BaseSessionAffinityProvider + { + private const string CustomHeaderNameKey = "CustomHeaderName"; + + public override string Mode => "CustomHeader"; + + //TBD. Add logging. + + protected override string GetDestinationAffinityKey(DestinationInfo destination) + { + return destination.DestinationId; + } + + protected override string GetRequestAffinityKey(HttpContext context, BackendConfig.BackendSessionAffinityOptions options) + { + var customHeaderName = GetSettingValue(CustomHeaderNameKey, options); + var keyHeaderValues = context.Request.Headers[customHeaderName]; + return !StringValues.IsNullOrEmpty(keyHeaderValues) ? keyHeaderValues[0] : null; // We always take the first value of a custom header storing an affinity key + } + + protected override void SetEncryptedAffinityKey(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, string encryptedKey) + { + var cookieName = GetSettingValue(CustomHeaderNameKey, options); + // TBD. The affinity key must be encrypted. + context.Response.Cookies.Append(cookieName, encryptedKey); + } + } +} diff --git a/src/ReverseProxy/Service/SessionAffinity/ISessionAffinityFeature.cs b/src/ReverseProxy/Service/SessionAffinity/ISessionAffinityFeature.cs deleted file mode 100644 index 7a8c36046..000000000 --- a/src/ReverseProxy/Service/SessionAffinity/ISessionAffinityFeature.cs +++ /dev/null @@ -1,26 +0,0 @@ -using Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract; -using Microsoft.ReverseProxy.RuntimeModel; - -namespace Microsoft.ReverseProxy.Service.SessionAffinity -{ - /// - /// Tracks current the request's affinity. - /// - public interface ISessionAffinityFeature - { - /// - /// Key binding the current request to one or many . - /// - public string DestinationKey { get; set; } - - /// - /// Affinity mode. - /// - public SessionAffinityMode Mode { get; set; } - - /// - /// Name of a custom header storing an affinity key to be used when is set to . - /// - public string CustomHeaderName { get; set; } - } -} diff --git a/src/ReverseProxy/Service/SessionAffinity/ISessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/ISessionAffinityProvider.cs index 1efd5dd81..5bd32a5b4 100644 --- a/src/ReverseProxy/Service/SessionAffinity/ISessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/ISessionAffinityProvider.cs @@ -13,14 +13,20 @@ namespace Microsoft.ReverseProxy.Service.SessionAffinity /// internal interface ISessionAffinityProvider { + /// + /// Session affinity mode this type provides. + /// + public string Mode { get; } + /// /// Tries to find to which the current request is affinitized by the affinity key. /// /// Current request's context. /// s available for the request. /// Affinity options. - /// . - public AffinitizedDestinationCollection TryFindAffinitizedDestinations(HttpContext context, IReadOnlyList destinations, BackendConfig.BackendSessionAffinityOptions options); + /// Affinitized s found for the request. + /// if affinitized s were successfully found, otherwise . + public bool TryFindAffinitizedDestinations(HttpContext context, IReadOnlyList destinations, BackendConfig.BackendSessionAffinityOptions options, out AffinitizedDestinationCollection affinitizedDestinations); /// /// Affinitize the current request to the given by setting the affinity key extracted from . @@ -29,11 +35,5 @@ internal interface ISessionAffinityProvider /// Affinity options. /// to which request is to be affinitized. public void AffinitizeRequest(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, DestinationInfo destination); - - /// - /// Sets an affinity key on a downstream response if any is defined by . - /// - /// Request context. - public void SetAffinityKeyOnDownstreamResponse(HttpContext context); } } diff --git a/src/ReverseProxy/Service/SessionAffinity/SessionAffinityFeature.cs b/src/ReverseProxy/Service/SessionAffinity/SessionAffinityFeature.cs deleted file mode 100644 index 928e70a8b..000000000 --- a/src/ReverseProxy/Service/SessionAffinity/SessionAffinityFeature.cs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract; - -namespace Microsoft.ReverseProxy.Service.SessionAffinity -{ - internal class SessionAffinityFeature : ISessionAffinityFeature - { - /// - public string DestinationKey { get; set; } - - /// - public SessionAffinityMode Mode { get; set; } - - /// - public string CustomHeaderName { get; set; } - } -} diff --git a/src/ReverseProxy/Service/SessionAffinity/SessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/SessionAffinityProvider.cs deleted file mode 100644 index 1ad1c7009..000000000 --- a/src/ReverseProxy/Service/SessionAffinity/SessionAffinityProvider.cs +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System; -using System.Collections.Generic; -using Microsoft.AspNetCore.Http; -using Microsoft.Extensions.Primitives; -using Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract; -using Microsoft.ReverseProxy.RuntimeModel; - -namespace Microsoft.ReverseProxy.Service.SessionAffinity -{ - internal class SessionAffinityProvider : ISessionAffinityProvider - { - private const string CookieName = "ms-rev-proxy-sess-key"; - - //TBD. Add logging. - - public void AffinitizeRequest(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, DestinationInfo destination) - { - if (!options.Enabled) - { - return; - } - - var affinityKey = destination.DestinationId; // Currently, we always use ID as the affinity key - - context.Features.Set(new SessionAffinityFeature { DestinationKey = affinityKey, Mode = options.Mode, CustomHeaderName = options.CustomHeaderName }); - } - - public AffinitizedDestinationCollection TryFindAffinitizedDestinations(HttpContext context, IReadOnlyList destinations, BackendConfig.BackendSessionAffinityOptions options) - { - if (!options.Enabled || destinations.Count == 0) - { - return default; - } - - var requestAffinityKey = GetAffinityKey(context, options); - - // TBD. Support different failure modes - if (requestAffinityKey == null) - { - return default; - } - - context.Features.Set(new SessionAffinityFeature { DestinationKey = requestAffinityKey, Mode = options.Mode, CustomHeaderName = options.CustomHeaderName }); - - // It's allowed to affinitize a request to a pool of destinations so as to enable load-balancing among them - var matchingDestinations = new List(); - for(var i = 0; i < destinations.Count; i++) - { - if (destinations[i].DestinationId == requestAffinityKey) - { - matchingDestinations.Add(destinations[i]); - } - } - - return new AffinitizedDestinationCollection(matchingDestinations, requestAffinityKey); - } - - public void SetAffinityKeyOnDownstreamResponse(HttpContext context) - { - var affinityFeature = context.Features.Get(); - - if (affinityFeature == null) - { - return; - } - - // TBD. The affinity key must be encrypted. - switch (affinityFeature.Mode) - { - case SessionAffinityMode.Cookie: - context.Response.Cookies.Append(CookieName, affinityFeature.DestinationKey); - return; - case SessionAffinityMode.CustomHeader: - context.Response.Headers.Add(affinityFeature.CustomHeaderName, new StringValues(affinityFeature.DestinationKey)); - return; - } - } - - private string GetAffinityKey(HttpContext context, BackendConfig.BackendSessionAffinityOptions options) - { - switch (options.Mode) - { - case SessionAffinityMode.Cookie: - return context.Request.Cookies.TryGetValue(CookieName, out var keyInCookie) ? keyInCookie : null; - case SessionAffinityMode.CustomHeader: - var keyHeaderValues = context.Request.Headers[options.CustomHeaderName]; - return !StringValues.IsNullOrEmpty(keyHeaderValues) ? keyHeaderValues[0] : null; // We always take the first value of a custom header storing an affinity key - default: - throw new ArgumentOutOfRangeException(nameof(options), $"Unsupported value {options.Mode} of {nameof(SessionAffinityMode)}."); - } - } - } -} From 074572b5cb4580a8d73266297a9499b031889ce3 Mon Sep 17 00:00:00 2001 From: Alexander Nikolaev Date: Wed, 20 May 2020 17:45:53 +0200 Subject: [PATCH 04/30] - Separate affinity middleware setup methods - AffinitizeRequestMiddleware picks a random destination if there are multiple of them - Affinity key is not logged anymore - IOption is used for CookieAffinitySessionProvider --- samples/ReverseProxy.Sample/Startup.cs | 4 +++- .../CookieSessionAffinityProviderOptions.cs | 5 +---- .../IReverseProxyBuilderExtensions.cs | 1 - ...rseProxyIEndpointRouteBuilderExtensions.cs | 4 +++- .../Middleware/AffinitizeRequestMiddleware.cs | 17 +++++++++++------ .../AffinitizedDestinationLookupMiddleware.cs | 18 +++++++++--------- .../HttpContextFeaturesExtensions.cs | 2 +- .../Middleware/LoadBalancingMiddleware.cs | 2 +- .../ProxyMiddlewareAppBuilderExtensions.cs | 19 ++++++++++++++----- .../SessionAffinityMiddlewareHelper.cs | 2 +- ...inationCollection.cs => AffinityResult.cs} | 7 ++----- .../BaseSessionAffinityProvider.cs | 8 ++++---- .../CookieSessionAffinityProvider.cs | 5 +++-- .../ISessionAffinityProvider.cs | 6 +++--- 14 files changed, 56 insertions(+), 44 deletions(-) rename src/ReverseProxy/Service/SessionAffinity/{AffinitizedDestinationCollection.cs => AffinityResult.cs} (58%) diff --git a/samples/ReverseProxy.Sample/Startup.cs b/samples/ReverseProxy.Sample/Startup.cs index 4bbebf768..15920924f 100644 --- a/samples/ReverseProxy.Sample/Startup.cs +++ b/samples/ReverseProxy.Sample/Startup.cs @@ -62,7 +62,9 @@ public void Configure(IApplicationBuilder app) return next(); }); - proxyPipeline.UseProxyLoadBalancingWithSessionAffinity(); + proxyPipeline.UseAffinitizedDestinationLookup(); + proxyPipeline.UseProxyLoadBalancing(); + proxyPipeline.UseRequestAffinity(); }); }); } diff --git a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/CookieSessionAffinityProviderOptions.cs b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/CookieSessionAffinityProviderOptions.cs index be82f7999..5d89c27f1 100644 --- a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/CookieSessionAffinityProviderOptions.cs +++ b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/CookieSessionAffinityProviderOptions.cs @@ -15,8 +15,6 @@ public class CookieSessionAffinityProviderOptions public static readonly string DefaultCookieName = ".Microsoft.ReverseProxy.Affinity"; - public static readonly string DefaultCookiePath = "/"; - public CookieBuilder Cookie { get => _cookieBuilder; @@ -28,9 +26,8 @@ private class AffinityCookieBuilder : CookieBuilder public AffinityCookieBuilder() { Name = DefaultCookieName; - Path = DefaultCookiePath; SecurePolicy = CookieSecurePolicy.None; - SameSite = SameSiteMode.Lax; + SameSite = SameSiteMode.Unspecified; HttpOnly = true; IsEssential = false; } diff --git a/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs b/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs index d131bbc13..ef798f4ad 100644 --- a/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs +++ b/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs @@ -89,7 +89,6 @@ public static IReverseProxyBuilder AddBackgroundWorkers(this IReverseProxyBuilde public static IReverseProxyBuilder AddSessionAffinityProvider(this IReverseProxyBuilder builder) { - builder.Services.TryAddSingleton(); builder.Services.TryAddEnumerable(new[] { new ServiceDescriptor(typeof(ISessionAffinityProvider), typeof(CookieSessionAffinityProvider), ServiceLifetime.Singleton), new ServiceDescriptor(typeof(ISessionAffinityProvider), typeof(CustomHeaderSessionAffinityProvider), ServiceLifetime.Singleton) diff --git a/src/ReverseProxy/Configuration/ReverseProxyIEndpointRouteBuilderExtensions.cs b/src/ReverseProxy/Configuration/ReverseProxyIEndpointRouteBuilderExtensions.cs index f486e6ac9..c37da1717 100644 --- a/src/ReverseProxy/Configuration/ReverseProxyIEndpointRouteBuilderExtensions.cs +++ b/src/ReverseProxy/Configuration/ReverseProxyIEndpointRouteBuilderExtensions.cs @@ -22,7 +22,9 @@ public static void MapReverseProxy(this IEndpointRouteBuilder endpoints) { endpoints.MapReverseProxy(app => { - app.UseProxyLoadBalancingWithSessionAffinity(); + app.UseAffinitizedDestinationLookup(); + app.UseProxyLoadBalancing(); + app.UseRequestAffinity(); }); } diff --git a/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs b/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs index 8108b1b45..08b4e864b 100644 --- a/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs +++ b/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs @@ -16,8 +16,9 @@ namespace Microsoft.ReverseProxy.Middleware /// internal class AffinitizeRequestMiddleware { + private readonly Random _random = new Random(); private readonly RequestDelegate _next; - private readonly IDictionary _sessionAffinityProviders = new Dictionary(); + private readonly IDictionary _sessionAffinityProviders; private readonly ILogger _logger; public AffinitizeRequestMiddleware(RequestDelegate next, IEnumerable sessionAffinityProviders, ILogger logger) @@ -29,9 +30,8 @@ public AffinitizeRequestMiddleware(RequestDelegate next, IEnumerable 1) + { + Log.AttemptToAffinitizeMultipleDestinations(_logger, backend.BackendId); + destination = destinations[_random.Next(destinations.Count)]; // It's assumed that all of them match to the request's affinity key. + } var currentProvider = _sessionAffinityProviders.GetRequiredProvider(options.Mode); currentProvider.AffinitizeRequest(context, options, destination); diff --git a/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs b/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs index c25822a93..a0f2794f8 100644 --- a/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs +++ b/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs @@ -29,7 +29,7 @@ public AffinitizedDestinationLookupMiddleware(RequestDelegate next, IEnumerable< public Task Invoke(HttpContext context) { - var backend = context.GetRequiredBacked(); + var backend = context.GetRequiredBackend(); var destinationsFeature = context.GetRequiredDestinationFeature(); var destinations = destinationsFeature.Destinations; @@ -39,15 +39,15 @@ public Task Invoke(HttpContext context) if (options.Enabled) { var currentProvider = _sessionAffinityProviders.GetRequiredProvider(options.Mode); - if (currentProvider.TryFindAffinitizedDestinations(context, destinations, options, out var affinitizedDestinations)) + if (currentProvider.TryFindAffinitizedDestinations(context, destinations, options, out var affinityResult)) { - if (affinitizedDestinations.Destinations.Count > 0) + if (affinityResult.Destinations.Count > 0) { - destinations = affinitizedDestinations.Destinations; + destinations = affinityResult.Destinations; } else { - Log.AffinitizedDestinationIsNotFound(_logger, affinitizedDestinations.RequestKey?.ToString(), backend.BackendId); + Log.AffinitizedDestinationIsNotFound(_logger, backend.BackendId); context.Response.StatusCode = 503; return Task.CompletedTask; } @@ -59,14 +59,14 @@ public Task Invoke(HttpContext context) private static class Log { - private static readonly Action _affinitizedDestinationIsNotFound = LoggerMessage.Define( + private static readonly Action _affinitizedDestinationIsNotFound = LoggerMessage.Define( LogLevel.Warning, EventIds.AffinitizedDestinationIsNotFound, - "No destinations found for the affinitized request with key `{affinityKey}` on backend `{backendId}`."); + "No destinations found for the affinitized request on backend `{backendId}`."); - public static void AffinitizedDestinationIsNotFound(ILogger logger, string affinityKey, string backendId) + public static void AffinitizedDestinationIsNotFound(ILogger logger, string backendId) { - _affinitizedDestinationIsNotFound(logger, affinityKey, backendId, null); + _affinitizedDestinationIsNotFound(logger, backendId, null); } } } diff --git a/src/ReverseProxy/Middleware/HttpContextFeaturesExtensions.cs b/src/ReverseProxy/Middleware/HttpContextFeaturesExtensions.cs index 129d62654..a3fef3609 100644 --- a/src/ReverseProxy/Middleware/HttpContextFeaturesExtensions.cs +++ b/src/ReverseProxy/Middleware/HttpContextFeaturesExtensions.cs @@ -9,7 +9,7 @@ namespace Microsoft.ReverseProxy.Middleware { internal static class HttpContextFeaturesExtensions { - public static BackendInfo GetRequiredBacked(this HttpContext context) + public static BackendInfo GetRequiredBackend(this HttpContext context) { return context.Features.Get() ?? throw new InvalidOperationException("Backend unspecified."); } diff --git a/src/ReverseProxy/Middleware/LoadBalancingMiddleware.cs b/src/ReverseProxy/Middleware/LoadBalancingMiddleware.cs index 94466b575..8f3de31b0 100644 --- a/src/ReverseProxy/Middleware/LoadBalancingMiddleware.cs +++ b/src/ReverseProxy/Middleware/LoadBalancingMiddleware.cs @@ -35,7 +35,7 @@ public LoadBalancingMiddleware( public Task Invoke(HttpContext context) { - var backend = context.GetRequiredBacked(); + var backend = context.GetRequiredBackend(); var destinationsFeature = context.GetRequiredDestinationFeature(); var destinations = destinationsFeature.Destinations; diff --git a/src/ReverseProxy/Middleware/ProxyMiddlewareAppBuilderExtensions.cs b/src/ReverseProxy/Middleware/ProxyMiddlewareAppBuilderExtensions.cs index 9095db82b..c4378514f 100644 --- a/src/ReverseProxy/Middleware/ProxyMiddlewareAppBuilderExtensions.cs +++ b/src/ReverseProxy/Middleware/ProxyMiddlewareAppBuilderExtensions.cs @@ -19,13 +19,22 @@ public static IApplicationBuilder UseProxyLoadBalancing(this IApplicationBuilder } /// - /// Load balances across the available endpoints and maintains session affinity. + /// Looks up one or multiple destinations affinitized to the request by an affinity key. + /// It only find affinitized destinations, but do not actually routes request to any of them. + /// Instead destinations are passed further to pipeline for load balancing or other processing steps. /// - public static IApplicationBuilder UseProxyLoadBalancingWithSessionAffinity(this IApplicationBuilder builder) + public static IApplicationBuilder UseAffinitizedDestinationLookup(this IApplicationBuilder builder) { - return builder.UseMiddleware() - .UseMiddleware() - .UseMiddleware(); + return builder.UseMiddleware(); + } + + /// + /// Routes the request to an affinitized destination looked up on previous steps. + /// If there are multiple affinitized destinations found for the request, it randomly picks one of them. + /// + public static IApplicationBuilder UseRequestAffinity(this IApplicationBuilder builder) + { + return builder.UseMiddleware(); } } } diff --git a/src/ReverseProxy/Middleware/SessionAffinityMiddlewareHelper.cs b/src/ReverseProxy/Middleware/SessionAffinityMiddlewareHelper.cs index 68bad18fe..95d9bc6b8 100644 --- a/src/ReverseProxy/Middleware/SessionAffinityMiddlewareHelper.cs +++ b/src/ReverseProxy/Middleware/SessionAffinityMiddlewareHelper.cs @@ -16,7 +16,7 @@ public static IDictionary ToProviderDictionary throw new ArgumentNullException(nameof(sessionAffinityProviders)); } - var result = new Dictionary(); + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); foreach (var provider in sessionAffinityProviders) { diff --git a/src/ReverseProxy/Service/SessionAffinity/AffinitizedDestinationCollection.cs b/src/ReverseProxy/Service/SessionAffinity/AffinityResult.cs similarity index 58% rename from src/ReverseProxy/Service/SessionAffinity/AffinitizedDestinationCollection.cs rename to src/ReverseProxy/Service/SessionAffinity/AffinityResult.cs index f15b71c97..236522965 100644 --- a/src/ReverseProxy/Service/SessionAffinity/AffinitizedDestinationCollection.cs +++ b/src/ReverseProxy/Service/SessionAffinity/AffinityResult.cs @@ -6,16 +6,13 @@ namespace Microsoft.ReverseProxy.Service.SessionAffinity { - public readonly struct AffinitizedDestinationCollection + public readonly struct AffinityResult { public readonly IReadOnlyList Destinations; - public readonly object RequestKey; - - public AffinitizedDestinationCollection(IReadOnlyList destinations, object requestKey) + public AffinityResult(IReadOnlyList destinations) { Destinations = destinations; - RequestKey = requestKey; } } } diff --git a/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs index df6c13b8f..8fe996c2a 100644 --- a/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs @@ -30,11 +30,11 @@ public virtual void AffinitizeRequest(HttpContext context, BackendConfig.Backend SetEncryptedAffinityKey(context, options, encryptedKey); } - public virtual bool TryFindAffinitizedDestinations(HttpContext context, IReadOnlyList destinations, BackendConfig.BackendSessionAffinityOptions options, out AffinitizedDestinationCollection affinitizedDestinations) + public virtual bool TryFindAffinitizedDestinations(HttpContext context, IReadOnlyList destinations, BackendConfig.BackendSessionAffinityOptions options, out AffinityResult affinityResult) { if (!options.Enabled || destinations.Count == 0) { - affinitizedDestinations = default; + affinityResult = default; return false; } @@ -43,7 +43,7 @@ public virtual bool TryFindAffinitizedDestinations(HttpContext context, IReadOnl // TBD. Support different failure modes if (requestAffinityKey == null) { - affinitizedDestinations = default; + affinityResult = default; return false; } @@ -59,7 +59,7 @@ public virtual bool TryFindAffinitizedDestinations(HttpContext context, IReadOnl } } - affinitizedDestinations = new AffinitizedDestinationCollection(matchingDestinations, requestAffinityKey); + affinityResult = new AffinityResult(matchingDestinations); return true; } diff --git a/src/ReverseProxy/Service/SessionAffinity/CookieSessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/CookieSessionAffinityProvider.cs index 407269fbb..6290639cc 100644 --- a/src/ReverseProxy/Service/SessionAffinity/CookieSessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/CookieSessionAffinityProvider.cs @@ -3,6 +3,7 @@ using System; using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Options; using Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract; using Microsoft.ReverseProxy.RuntimeModel; @@ -12,9 +13,9 @@ internal class CookieSessionAffinityProvider : BaseSessionAffinityProvider providerOptions) { - _providerOptions = providerOptions ?? throw new ArgumentNullException(nameof(providerOptions)); + _providerOptions = providerOptions?.Value ?? throw new ArgumentNullException(nameof(providerOptions)); } public override string Mode => "Cookie"; diff --git a/src/ReverseProxy/Service/SessionAffinity/ISessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/ISessionAffinityProvider.cs index 5bd32a5b4..97fc1298c 100644 --- a/src/ReverseProxy/Service/SessionAffinity/ISessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/ISessionAffinityProvider.cs @@ -14,7 +14,7 @@ namespace Microsoft.ReverseProxy.Service.SessionAffinity internal interface ISessionAffinityProvider { /// - /// Session affinity mode this type provides. + /// A unique identifier for this session affinity implementation. This will be referenced from config. /// public string Mode { get; } @@ -24,9 +24,9 @@ internal interface ISessionAffinityProvider /// Current request's context. /// s available for the request. /// Affinity options. - /// Affinitized s found for the request. + /// Affinitized s found for the request. /// if affinitized s were successfully found, otherwise . - public bool TryFindAffinitizedDestinations(HttpContext context, IReadOnlyList destinations, BackendConfig.BackendSessionAffinityOptions options, out AffinitizedDestinationCollection affinitizedDestinations); + public bool TryFindAffinitizedDestinations(HttpContext context, IReadOnlyList destinations, BackendConfig.BackendSessionAffinityOptions options, out AffinityResult affinityResult); /// /// Affinitize the current request to the given by setting the affinity key extracted from . From 55286d1c0a8eab85ca1f306fa422e37ec881f337 Mon Sep 17 00:00:00 2001 From: Alexander Nikolaev Date: Fri, 22 May 2020 14:50:31 +0200 Subject: [PATCH 05/30] - Missing affinitized destination failure handling - Affinity key data protection - Session affinity configuration validation - Fixed comments - Extra logging --- samples/ReverseProxy.Sample/Startup.cs | 2 +- samples/ReverseProxy.Sample/appsettings.json | 4 ++ .../Contract/SessionAffinityOptions.cs | 6 +++ .../IReverseProxyBuilderExtensions.cs | 17 +++++++ ...ReverseProxyServiceCollectionExtensions.cs | 1 + ...rseProxyIEndpointRouteBuilderExtensions.cs | 2 +- .../Middleware/AffinitizeRequestMiddleware.cs | 41 ++++++++------- .../AffinitizedDestinationLookupMiddleware.cs | 29 ++++++++--- .../ProxyMiddlewareAppBuilderExtensions.cs | 12 +++-- .../SessionAffinityMiddlewareHelper.cs | 41 --------------- .../Service/Config/ConfigErrors.cs | 4 ++ .../Service/Config/DynamicConfigBuilder.cs | 43 ++++++++++++++-- .../Management/ReverseProxyConfigManager.cs | 5 +- .../Service/RuntimeModel/BackendConfig.cs | 7 ++- .../BaseSessionAffinityProvider.cs | 29 ++++++++--- .../CookieSessionAffinityProvider.cs | 18 ++++--- .../CustomHeaderSessionAffinityProvider.cs | 18 ++++--- .../IMissingDestinationHandler.cs | 29 +++++++++++ .../PickRandomMissingDestinationHandler.cs | 28 ++++++++++ .../ReturnErrorMissingDestinationHandler.cs | 19 +++++++ .../SessionAffinityMiddlewareHelper.cs | 51 +++++++++++++++++++ 21 files changed, 308 insertions(+), 98 deletions(-) delete mode 100644 src/ReverseProxy/Middleware/SessionAffinityMiddlewareHelper.cs create mode 100644 src/ReverseProxy/Service/SessionAffinity/IMissingDestinationHandler.cs create mode 100644 src/ReverseProxy/Service/SessionAffinity/PickRandomMissingDestinationHandler.cs create mode 100644 src/ReverseProxy/Service/SessionAffinity/ReturnErrorMissingDestinationHandler.cs create mode 100644 src/ReverseProxy/Service/SessionAffinity/SessionAffinityMiddlewareHelper.cs diff --git a/samples/ReverseProxy.Sample/Startup.cs b/samples/ReverseProxy.Sample/Startup.cs index 15920924f..42f9aa178 100644 --- a/samples/ReverseProxy.Sample/Startup.cs +++ b/samples/ReverseProxy.Sample/Startup.cs @@ -64,7 +64,7 @@ public void Configure(IApplicationBuilder app) }); proxyPipeline.UseAffinitizedDestinationLookup(); proxyPipeline.UseProxyLoadBalancing(); - proxyPipeline.UseRequestAffinity(); + proxyPipeline.UseRequestAffinitizer(); }); }); } diff --git a/samples/ReverseProxy.Sample/appsettings.json b/samples/ReverseProxy.Sample/appsettings.json index c42715bbb..8ee0cc0d8 100644 --- a/samples/ReverseProxy.Sample/appsettings.json +++ b/samples/ReverseProxy.Sample/appsettings.json @@ -23,6 +23,10 @@ "Metadata": { "CustomHealth": "false" }, + "SessionAffinity": { + "Mode": "Cookie", + "MissingDestinationHandler": "ReturnError" + }, "Destinations": { "backend1/destination1": { "Address": "https://localhost:10000/" diff --git a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityOptions.cs b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityOptions.cs index d7d1d151f..22ef60256 100644 --- a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityOptions.cs +++ b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityOptions.cs @@ -16,6 +16,11 @@ public sealed class SessionAffinityOptions /// public string Mode { get; set; } + /// + /// Strategy handling missing destination for an affinitized request. + /// + public string MissingDestinationHandler { get; set; } + /// /// Key-value pair collection holding extra settings specific to different affinity modes. /// @@ -26,6 +31,7 @@ internal SessionAffinityOptions DeepClone() return new SessionAffinityOptions { Mode = Mode, + MissingDestinationHandler = MissingDestinationHandler, Settings = Settings?.DeepClone(StringComparer.Ordinal) }; } diff --git a/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs b/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs index ef798f4ad..c6242713f 100644 --- a/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs +++ b/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using System; +using Microsoft.AspNetCore.DataProtection; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.ReverseProxy.Abstractions; @@ -89,6 +90,10 @@ public static IReverseProxyBuilder AddBackgroundWorkers(this IReverseProxyBuilde public static IReverseProxyBuilder AddSessionAffinityProvider(this IReverseProxyBuilder builder) { + builder.Services.TryAddEnumerable(new[] { + new ServiceDescriptor(typeof(IMissingDestinationHandler), typeof(PickRandomMissingDestinationHandler), ServiceLifetime.Singleton), + new ServiceDescriptor(typeof(IMissingDestinationHandler), typeof(ReturnErrorMissingDestinationHandler), ServiceLifetime.Singleton) + }); builder.Services.TryAddEnumerable(new[] { new ServiceDescriptor(typeof(ISessionAffinityProvider), typeof(CookieSessionAffinityProvider), ServiceLifetime.Singleton), new ServiceDescriptor(typeof(ISessionAffinityProvider), typeof(CustomHeaderSessionAffinityProvider), ServiceLifetime.Singleton) @@ -96,5 +101,17 @@ public static IReverseProxyBuilder AddSessionAffinityProvider(this IReverseProxy return builder; } + + public static IReverseProxyBuilder AddDataProtection(this IReverseProxyBuilder builder) + { + builder.Services.AddDataProtection(); + return builder; + } + + public static IReverseProxyBuilder AddDataProtection(this IReverseProxyBuilder builder, Action setupAction) + { + builder.Services.AddDataProtection(setupAction); + return builder; + } } } diff --git a/src/ReverseProxy/Configuration/DependencyInjection/ReverseProxyServiceCollectionExtensions.cs b/src/ReverseProxy/Configuration/DependencyInjection/ReverseProxyServiceCollectionExtensions.cs index a4462253d..d99b578be 100644 --- a/src/ReverseProxy/Configuration/DependencyInjection/ReverseProxyServiceCollectionExtensions.cs +++ b/src/ReverseProxy/Configuration/DependencyInjection/ReverseProxyServiceCollectionExtensions.cs @@ -29,6 +29,7 @@ public static IReverseProxyBuilder AddReverseProxy(this IServiceCollection servi .AddRuntimeStateManagers() .AddConfigManager() .AddDynamicEndpointDataSource() + .AddDataProtection() .AddSessionAffinityProvider() .AddProxy() .AddBackgroundWorkers(); diff --git a/src/ReverseProxy/Configuration/ReverseProxyIEndpointRouteBuilderExtensions.cs b/src/ReverseProxy/Configuration/ReverseProxyIEndpointRouteBuilderExtensions.cs index c37da1717..d79081adc 100644 --- a/src/ReverseProxy/Configuration/ReverseProxyIEndpointRouteBuilderExtensions.cs +++ b/src/ReverseProxy/Configuration/ReverseProxyIEndpointRouteBuilderExtensions.cs @@ -24,7 +24,7 @@ public static void MapReverseProxy(this IEndpointRouteBuilder endpoints) { app.UseAffinitizedDestinationLookup(); app.UseProxyLoadBalancing(); - app.UseRequestAffinity(); + app.UseRequestAffinitizer(); }); } diff --git a/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs b/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs index 08b4e864b..f19f0facb 100644 --- a/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs +++ b/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; +using Microsoft.ReverseProxy.Abstractions.Telemetry; using Microsoft.ReverseProxy.RuntimeModel; using Microsoft.ReverseProxy.Service.SessionAffinity; @@ -19,13 +20,19 @@ internal class AffinitizeRequestMiddleware private readonly Random _random = new Random(); private readonly RequestDelegate _next; private readonly IDictionary _sessionAffinityProviders; + private readonly IOperationLogger _operationLogger; private readonly ILogger _logger; - public AffinitizeRequestMiddleware(RequestDelegate next, IEnumerable sessionAffinityProviders, ILogger logger) + public AffinitizeRequestMiddleware( + RequestDelegate next, + IEnumerable sessionAffinityProviders, + IOperationLogger operationLogger, + ILogger logger) { _next = next ?? throw new ArgumentNullException(nameof(next)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _sessionAffinityProviders = sessionAffinityProviders.ToProviderDictionary(); + _operationLogger = operationLogger ?? throw new ArgumentNullException(nameof(operationLogger)); } public Task Invoke(HttpContext context) @@ -36,25 +43,25 @@ public Task Invoke(HttpContext context) if (options.Enabled) { var destinationsFeature = context.GetRequiredDestinationFeature(); + var destination = _operationLogger.Execute("ReverseProxy.AffinitizeRequest", () => AffinitizeRequest(context, backend, options, destinationsFeature.Destinations)); + destinationsFeature.Destinations = new[] { destination }; + } - if (destinationsFeature.Destinations.Count > 1) - { - Log.AttemptToAffinitizeMultipleDestinations(_logger, backend.BackendId); - } - - var destinations = destinationsFeature.Destinations; - var destination = destinations[0]; - if (destinations.Count > 1) - { - Log.AttemptToAffinitizeMultipleDestinations(_logger, backend.BackendId); - destination = destinations[_random.Next(destinations.Count)]; // It's assumed that all of them match to the request's affinity key. - } + return _next(context); + } - var currentProvider = _sessionAffinityProviders.GetRequiredProvider(options.Mode); - currentProvider.AffinitizeRequest(context, options, destination); + private DestinationInfo AffinitizeRequest(HttpContext context, BackendInfo backend, BackendConfig.BackendSessionAffinityOptions options, IReadOnlyList destinations) + { + var destination = destinations[0]; + if (destinations.Count > 1) + { + Log.AttemptToAffinitizeMultipleDestinations(_logger, backend.BackendId); + destination = destinations[_random.Next(destinations.Count)]; // It's assumed that all of them match to the request's affinity key. } - return _next(context); + var currentProvider = _sessionAffinityProviders.GetRequiredServiceById(options.Mode); + currentProvider.AffinitizeRequest(context, options, destination); + return destination; } private static class Log @@ -62,7 +69,7 @@ private static class Log private static readonly Action _attemptToAffinitizeMultipleDestinations = LoggerMessage.Define( LogLevel.Warning, EventIds.AttemptToAffinitizeMultipleDestinations, - "Attempt to affinitize multiple destinations to the same request on backend `{backendId}`. The first destination will be used."); + "The request still has multiple destinations on the backend {backendId} to choose from when establishing affinity, load balancing may not be properly configured. A random destination will be used."); public static void AttemptToAffinitizeMultipleDestinations(ILogger logger, string backendId) { diff --git a/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs b/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs index a0f2794f8..3f2f18cd4 100644 --- a/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs +++ b/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; +using Microsoft.ReverseProxy.Abstractions.Telemetry; using Microsoft.ReverseProxy.RuntimeModel; using Microsoft.ReverseProxy.Service.SessionAffinity; @@ -18,12 +19,18 @@ internal class AffinitizedDestinationLookupMiddleware { private readonly RequestDelegate _next; private readonly IDictionary _sessionAffinityProviders = new Dictionary(); + private readonly IOperationLogger _operationLogger; private readonly ILogger _logger; - public AffinitizedDestinationLookupMiddleware(RequestDelegate next, IEnumerable sessionAffinityProviders, ILogger logger) + public AffinitizedDestinationLookupMiddleware( + RequestDelegate next, + IEnumerable sessionAffinityProviders, + IOperationLogger operationLogger, + ILogger logger) { _next = next ?? throw new ArgumentNullException(nameof(next)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _operationLogger = operationLogger ?? throw new ArgumentNullException(nameof(logger)); _sessionAffinityProviders = sessionAffinityProviders.ToProviderDictionary(); } @@ -34,16 +41,19 @@ public Task Invoke(HttpContext context) var destinations = destinationsFeature.Destinations; var options = backend.Config.Value?.SessionAffinityOptions - ?? new BackendConfig.BackendSessionAffinityOptions(false, default, default); + ?? new BackendConfig.BackendSessionAffinityOptions(false, default, default, default); if (options.Enabled) { - var currentProvider = _sessionAffinityProviders.GetRequiredProvider(options.Mode); - if (currentProvider.TryFindAffinitizedDestinations(context, destinations, options, out var affinityResult)) + var affinitizedDestinations = _operationLogger.Execute( + "ReverseProxy.FindAffinitizedDestinations", + () => FindAffinitizedDestinations(context, destinations, options)); + if (affinitizedDestinations.DestinationsFound) { - if (affinityResult.Destinations.Count > 0) + if (affinitizedDestinations.Result.Destinations.Count > 0) { - destinations = affinityResult.Destinations; + destinations = affinitizedDestinations.Result.Destinations; + destinationsFeature.Destinations = destinations; } else { @@ -57,6 +67,13 @@ public Task Invoke(HttpContext context) return _next(context); } + private (bool DestinationsFound, AffinityResult Result) FindAffinitizedDestinations(HttpContext context, IReadOnlyList destinations, BackendConfig.BackendSessionAffinityOptions options) + { + var currentProvider = _sessionAffinityProviders.GetRequiredServiceById(options.Mode); + var destinationsFound = currentProvider.TryFindAffinitizedDestinations(context, destinations, options, out var affinityResult); + return (destinationsFound, affinityResult); + } + private static class Log { private static readonly Action _affinitizedDestinationIsNotFound = LoggerMessage.Define( diff --git a/src/ReverseProxy/Middleware/ProxyMiddlewareAppBuilderExtensions.cs b/src/ReverseProxy/Middleware/ProxyMiddlewareAppBuilderExtensions.cs index c4378514f..5c5285a7c 100644 --- a/src/ReverseProxy/Middleware/ProxyMiddlewareAppBuilderExtensions.cs +++ b/src/ReverseProxy/Middleware/ProxyMiddlewareAppBuilderExtensions.cs @@ -19,9 +19,10 @@ public static IApplicationBuilder UseProxyLoadBalancing(this IApplicationBuilder } /// - /// Looks up one or multiple destinations affinitized to the request by an affinity key. - /// It only find affinitized destinations, but do not actually routes request to any of them. - /// Instead destinations are passed further to pipeline for load balancing or other processing steps. + /// Checks if a request has an established affinity relationship and if the associated destination is available. + /// This should be placed before load balancing and other destination selection components. + /// Requests without an affinity relationship will be a processed normally and have the affinity relationship + /// established by a later component. See . /// public static IApplicationBuilder UseAffinitizedDestinationLookup(this IApplicationBuilder builder) { @@ -29,10 +30,11 @@ public static IApplicationBuilder UseAffinitizedDestinationLookup(this IApplicat } /// - /// Routes the request to an affinitized destination looked up on previous steps. + /// Establishes the affinity relationship to the selected destination. /// If there are multiple affinitized destinations found for the request, it randomly picks one of them. + /// This should be placed after load balancing and other destination selection processes. /// - public static IApplicationBuilder UseRequestAffinity(this IApplicationBuilder builder) + public static IApplicationBuilder UseRequestAffinitizer(this IApplicationBuilder builder) { return builder.UseMiddleware(); } diff --git a/src/ReverseProxy/Middleware/SessionAffinityMiddlewareHelper.cs b/src/ReverseProxy/Middleware/SessionAffinityMiddlewareHelper.cs deleted file mode 100644 index 95d9bc6b8..000000000 --- a/src/ReverseProxy/Middleware/SessionAffinityMiddlewareHelper.cs +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System; -using System.Collections.Generic; -using Microsoft.ReverseProxy.Service.SessionAffinity; - -namespace Microsoft.ReverseProxy.Middleware -{ - internal static class SessionAffinityMiddlewareHelper - { - public static IDictionary ToProviderDictionary(this IEnumerable sessionAffinityProviders) - { - if (sessionAffinityProviders == null) - { - throw new ArgumentNullException(nameof(sessionAffinityProviders)); - } - - var result = new Dictionary(StringComparer.OrdinalIgnoreCase); - - foreach (var provider in sessionAffinityProviders) - { - if (!result.TryAdd(provider.Mode, provider)) - { - throw new ArgumentException(nameof(sessionAffinityProviders), $"More than one {nameof(ISessionAffinityProvider)} found with the same Mode value."); - } - } - - return result; - } - - public static ISessionAffinityProvider GetRequiredProvider(this IDictionary sessionAffinityProviders, string mode) - { - if (!sessionAffinityProviders.TryGetValue(mode, out var currentProvider)) - { - throw new ArgumentException(nameof(mode), $"No {nameof(ISessionAffinityProvider)} was found for the mode {mode}."); - } - return currentProvider; - } - } -} diff --git a/src/ReverseProxy/Service/Config/ConfigErrors.cs b/src/ReverseProxy/Service/Config/ConfigErrors.cs index 79e0ba151..241e12039 100644 --- a/src/ReverseProxy/Service/Config/ConfigErrors.cs +++ b/src/ReverseProxy/Service/Config/ConfigErrors.cs @@ -23,6 +23,10 @@ internal static class ConfigErrors internal const string ParsedRouteRuleInvalidMatcher = "ParsedRoute_RuleInvalidMatcher"; internal const string ConfigBuilderBackendIdMismatch = "ConfigBuilder_BackendIdMismatch"; + internal const string ConfigBuilderBackendSessionAffinityModeIsNull = "ConfigBuilder_BackendSessionAffinityModeIsNull"; + internal const string ConfigBuilderBackendNoProviderFoundForSessionAffinityMode = "ConfigBuilder_BackendNoProviderFoundForSessionAffinityMode"; + internal const string ConfigBuilderBackendMissingDestinationHandlerIsNull = "ConfigBuilder_MissingDestinationHandlerIsNull"; + internal const string ConfigBuilderBackendNoMissingDestinationHandlerFoundForSpecifiedName = "ConfigBuilder_NoMissingDestinationHandlerFoundForSpecifiedName"; internal const string ConfigBuilderBackendException = "ConfigBuilder_BackendException"; internal const string ConfigBuilderRouteException = "ConfigBuilder_RouteException"; } diff --git a/src/ReverseProxy/Service/Config/DynamicConfigBuilder.cs b/src/ReverseProxy/Service/Config/DynamicConfigBuilder.cs index 7c7c0a4fb..8aaed8e05 100644 --- a/src/ReverseProxy/Service/Config/DynamicConfigBuilder.cs +++ b/src/ReverseProxy/Service/Config/DynamicConfigBuilder.cs @@ -3,12 +3,11 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.Threading; using System.Threading.Tasks; -using Microsoft.Extensions.Options; using Microsoft.ReverseProxy.Abstractions; using Microsoft.ReverseProxy.ConfigModel; +using Microsoft.ReverseProxy.Service.SessionAffinity; using Microsoft.ReverseProxy.Utilities; namespace Microsoft.ReverseProxy.Service @@ -19,21 +18,29 @@ internal class DynamicConfigBuilder : IDynamicConfigBuilder private readonly IBackendsRepo _backendsRepo; private readonly IRoutesRepo _routesRepo; private readonly IRouteValidator _parsedRouteValidator; + private readonly IDictionary _sessionAffinityProviders; + private readonly IDictionary _missingDestionationHandlers; public DynamicConfigBuilder( IEnumerable filters, IBackendsRepo backendsRepo, IRoutesRepo routesRepo, - IRouteValidator parsedRouteValidator) + IRouteValidator parsedRouteValidator, + IEnumerable sessionAffinityProviders, + IEnumerable missingDestinationHandlers) { Contracts.CheckValue(filters, nameof(filters)); Contracts.CheckValue(backendsRepo, nameof(backendsRepo)); Contracts.CheckValue(routesRepo, nameof(routesRepo)); Contracts.CheckValue(parsedRouteValidator, nameof(parsedRouteValidator)); + Contracts.CheckValue(sessionAffinityProviders, nameof(sessionAffinityProviders)); + Contracts.CheckValue(missingDestinationHandlers, nameof(missingDestinationHandlers)); _filters = filters; _backendsRepo = backendsRepo; _routesRepo = routesRepo; _parsedRouteValidator = parsedRouteValidator; + _sessionAffinityProviders = sessionAffinityProviders.ToProviderDictionary(); + _missingDestionationHandlers = missingDestinationHandlers.ToHandlerDictionary(); } public async Task> BuildConfigAsync(IConfigErrorReporter errorReporter, CancellationToken cancellation) @@ -68,6 +75,8 @@ public async Task> GetBackendsAsync(IConfigErrorRep continue; } + ValidateSessionAffinity(errorReporter, id, backend); + foreach (var filter in _filters) { await filter.ConfigureBackendAsync(backend, cancellation); @@ -84,6 +93,34 @@ public async Task> GetBackendsAsync(IConfigErrorRep return configuredBackends; } + private void ValidateSessionAffinity(IConfigErrorReporter errorReporter, string id, Backend backend) + { + if (backend.SessionAffinity == null) // Session affinity is disabled + { + return; + } + + var affinityMode = backend.SessionAffinity.Mode; + if (affinityMode == null) + { + errorReporter.ReportError(ConfigErrors.ConfigBuilderBackendSessionAffinityModeIsNull, id, $"The session affinity mode is null for the backend {backend.Id}."); + } + else if (!_sessionAffinityProviders.ContainsKey(affinityMode)) + { + errorReporter.ReportError(ConfigErrors.ConfigBuilderBackendNoProviderFoundForSessionAffinityMode, id, $"No matching {nameof(ISessionAffinityProvider)} found for the session affinity mode {affinityMode} set on the backend {backend.Id}."); + } + + var missingDestinationHandler = backend.SessionAffinity.MissingDestinationHandler; + if (missingDestinationHandler == null) + { + errorReporter.ReportError(ConfigErrors.ConfigBuilderBackendMissingDestinationHandlerIsNull, id, $"The missing affinitizated destination handler name is null for the backend {backend.Id}."); + } + else if (!_missingDestionationHandlers.ContainsKey(missingDestinationHandler)) + { + errorReporter.ReportError(ConfigErrors.ConfigBuilderBackendNoMissingDestinationHandlerFoundForSpecifiedName, id, $"No matching {nameof(IMissingDestinationHandler)} found for the missing affinitizated destination handler name {missingDestinationHandler} set on the backend {backend.Id}."); + } + } + private async Task> GetRoutesAsync(IConfigErrorReporter errorReporter, CancellationToken cancellation) { var routes = await _routesRepo.GetRoutesAsync(cancellation); diff --git a/src/ReverseProxy/Service/Management/ReverseProxyConfigManager.cs b/src/ReverseProxy/Service/Management/ReverseProxyConfigManager.cs index 0884c89ed..b3a22c6ff 100644 --- a/src/ReverseProxy/Service/Management/ReverseProxyConfigManager.cs +++ b/src/ReverseProxy/Service/Management/ReverseProxyConfigManager.cs @@ -97,8 +97,9 @@ private void UpdateRuntimeBackends(DynamicConfigRoot config) mode: configBackend.LoadBalancing?.Mode ?? default), new BackendConfig.BackendSessionAffinityOptions( enabled: configBackend.SessionAffinity != null, - mode: configBackend.SessionAffinity?.Mode ?? default, - settings: configBackend.SessionAffinity?.Settings)); + mode: configBackend.SessionAffinity?.Mode, + missingDestinationHandler: configBackend.SessionAffinity?.MissingDestinationHandler, + settings: configBackend.SessionAffinity?.Settings as IReadOnlyDictionary)); var currentBackendConfig = backend.Config.Value; if (currentBackendConfig == null || diff --git a/src/ReverseProxy/Service/RuntimeModel/BackendConfig.cs b/src/ReverseProxy/Service/RuntimeModel/BackendConfig.cs index 075001a00..fc5835ba8 100644 --- a/src/ReverseProxy/Service/RuntimeModel/BackendConfig.cs +++ b/src/ReverseProxy/Service/RuntimeModel/BackendConfig.cs @@ -97,10 +97,11 @@ public BackendLoadBalancingOptions(LoadBalancingMode mode) internal readonly struct BackendSessionAffinityOptions { - public BackendSessionAffinityOptions(bool enabled, string mode, IDictionary settings) + public BackendSessionAffinityOptions(bool enabled, string mode, string missingDestinationHandler, IReadOnlyDictionary settings) { Mode = mode; - Settings = (IReadOnlyDictionary) settings; // Assume that actual dictionary type always implements IReadOnlyDictionary + MissingDestinationHandler = missingDestinationHandler; + Settings = settings; Enabled = enabled; } @@ -108,6 +109,8 @@ public BackendSessionAffinityOptions(bool enabled, string mode, IDictionary Settings { get; } } } diff --git a/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs index 8fe996c2a..b314ad87e 100644 --- a/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using Microsoft.AspNetCore.DataProtection; using Microsoft.AspNetCore.Http; using Microsoft.ReverseProxy.RuntimeModel; @@ -11,6 +12,14 @@ namespace Microsoft.ReverseProxy.Service.SessionAffinity internal abstract class BaseSessionAffinityProvider : ISessionAffinityProvider { protected static readonly object AffinityKeyId = new object(); + protected readonly IDataProtector DataProtector; + protected readonly IDictionary MissingDestinationHandlers; + + protected BaseSessionAffinityProvider(IDataProtectionProvider dataProtectionProvider, IEnumerable missingDestinationHandlers) + { + DataProtector = dataProtectionProvider?.CreateProtector(GetType().FullName) ?? throw new ArgumentNullException(nameof(dataProtectionProvider)); + MissingDestinationHandlers = missingDestinationHandlers?.ToHandlerDictionary() ?? throw new ArgumentNullException(nameof(missingDestinationHandlers)); + } public abstract string Mode { get; } @@ -26,8 +35,7 @@ public virtual void AffinitizeRequest(HttpContext context, BackendConfig.Backend affinityKey = GetDestinationAffinityKey(destination); } - var encryptedKey = (string)affinityKey; // TBD. The affinity key must be encrypted. - SetEncryptedAffinityKey(context, options, encryptedKey); + SetAffinityKey(context, options, (T)affinityKey); } public virtual bool TryFindAffinitizedDestinations(HttpContext context, IReadOnlyList destinations, BackendConfig.BackendSessionAffinityOptions options, out AffinityResult affinityResult) @@ -40,7 +48,6 @@ public virtual bool TryFindAffinitizedDestinations(HttpContext context, IReadOnl var requestAffinityKey = GetRequestAffinityKey(context, options); - // TBD. Support different failure modes if (requestAffinityKey == null) { affinityResult = default; @@ -59,13 +66,23 @@ public virtual bool TryFindAffinitizedDestinations(HttpContext context, IReadOnl } } - affinityResult = new AffinityResult(matchingDestinations); + if (matchingDestinations.Count == 0) + { + var failureHandler = MissingDestinationHandlers[options.MissingDestinationHandler]; + var newAffinitizedDestinations = failureHandler.Handle(context, options, requestAffinityKey, destinations); + affinityResult = new AffinityResult(newAffinitizedDestinations); + } + else + { + affinityResult = new AffinityResult(matchingDestinations); + } + return true; } protected virtual string GetSettingValue(string key, BackendConfig.BackendSessionAffinityOptions options) { - if (options.Settings.TryGetValue(key, out var value)) + if (options.Settings == null || !options.Settings.TryGetValue(key, out var value)) { throw new ArgumentException(nameof(options), $"{nameof(CookieSessionAffinityProvider)} couldn't find the required parameter {key} in session affinity settings."); } @@ -77,6 +94,6 @@ protected virtual string GetSettingValue(string key, BackendConfig.BackendSessio protected abstract T GetRequestAffinityKey(HttpContext context, BackendConfig.BackendSessionAffinityOptions options); - protected abstract void SetEncryptedAffinityKey(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, string encryptedKey); + protected abstract void SetAffinityKey(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, T unencryptedKey); } } diff --git a/src/ReverseProxy/Service/SessionAffinity/CookieSessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/CookieSessionAffinityProvider.cs index 6290639cc..432bda715 100644 --- a/src/ReverseProxy/Service/SessionAffinity/CookieSessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/CookieSessionAffinityProvider.cs @@ -3,9 +3,11 @@ using System; using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.DataProtection; using Microsoft.Extensions.Options; using Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract; using Microsoft.ReverseProxy.RuntimeModel; +using System.Collections.Generic; namespace Microsoft.ReverseProxy.Service.SessionAffinity { @@ -13,15 +15,17 @@ internal class CookieSessionAffinityProvider : BaseSessionAffinityProvider providerOptions) + public CookieSessionAffinityProvider( + IOptions providerOptions, + IDataProtectionProvider dataProtectionProvider, + IEnumerable missingDestinationHandlers) + : base(dataProtectionProvider, missingDestinationHandlers) { _providerOptions = providerOptions?.Value ?? throw new ArgumentNullException(nameof(providerOptions)); } public override string Mode => "Cookie"; - //TBD. Add logging. - protected override string GetDestinationAffinityKey(DestinationInfo destination) { return destination.DestinationId; @@ -29,14 +33,14 @@ protected override string GetDestinationAffinityKey(DestinationInfo destination) protected override string GetRequestAffinityKey(HttpContext context, BackendConfig.BackendSessionAffinityOptions options) { - return context.Request.Cookies.TryGetValue(_providerOptions.Cookie.Name, out var keyInCookie) ? keyInCookie : null; + var encryptedRequestKey = context.Request.Cookies.TryGetValue(_providerOptions.Cookie.Name, out var keyInCookie) ? keyInCookie : null; + return encryptedRequestKey != null ? DataProtector.Unprotect(encryptedRequestKey) : null; } - protected override void SetEncryptedAffinityKey(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, string encryptedKey) + protected override void SetAffinityKey(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, string unencryptedKey) { var affinityCookieOptions = _providerOptions.Cookie.Build(context); - // TBD. The affinity key must be encrypted. - context.Response.Cookies.Append(_providerOptions.Cookie.Name, encryptedKey, affinityCookieOptions); + context.Response.Cookies.Append(_providerOptions.Cookie.Name, DataProtector.Protect(unencryptedKey), affinityCookieOptions); } } } diff --git a/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs index b9d7f5ff1..fba142340 100644 --- a/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System.Collections.Generic; +using Microsoft.AspNetCore.DataProtection; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Primitives; using Microsoft.ReverseProxy.RuntimeModel; @@ -11,9 +13,11 @@ internal class CustomHeaderSessionAffinityProvider : BaseSessionAffinityProvider { private const string CustomHeaderNameKey = "CustomHeaderName"; - public override string Mode => "CustomHeader"; + public CustomHeaderSessionAffinityProvider(IDataProtectionProvider dataProtectionProvider, IEnumerable missingDestinationHandlers) + : base(dataProtectionProvider, missingDestinationHandlers) + {} - //TBD. Add logging. + public override string Mode => "CustomHeader"; protected override string GetDestinationAffinityKey(DestinationInfo destination) { @@ -24,14 +28,14 @@ protected override string GetRequestAffinityKey(HttpContext context, BackendConf { var customHeaderName = GetSettingValue(CustomHeaderNameKey, options); var keyHeaderValues = context.Request.Headers[customHeaderName]; - return !StringValues.IsNullOrEmpty(keyHeaderValues) ? keyHeaderValues[0] : null; // We always take the first value of a custom header storing an affinity key + var encryptedRequestKey = !StringValues.IsNullOrEmpty(keyHeaderValues) ? keyHeaderValues[0] : null; // We always take the first value of a custom header storing an affinity key + return encryptedRequestKey != null ? DataProtector.Unprotect(encryptedRequestKey) : null; } - protected override void SetEncryptedAffinityKey(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, string encryptedKey) + protected override void SetAffinityKey(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, string unencryptedKey) { - var cookieName = GetSettingValue(CustomHeaderNameKey, options); - // TBD. The affinity key must be encrypted. - context.Response.Cookies.Append(cookieName, encryptedKey); + var customHeaderName = GetSettingValue(CustomHeaderNameKey, options); + context.Response.Headers.Append(customHeaderName, DataProtector.Protect(unencryptedKey)); } } } diff --git a/src/ReverseProxy/Service/SessionAffinity/IMissingDestinationHandler.cs b/src/ReverseProxy/Service/SessionAffinity/IMissingDestinationHandler.cs new file mode 100644 index 000000000..4e19f357a --- /dev/null +++ b/src/ReverseProxy/Service/SessionAffinity/IMissingDestinationHandler.cs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Collections.Generic; +using Microsoft.AspNetCore.Http; +using Microsoft.ReverseProxy.RuntimeModel; + +namespace Microsoft.ReverseProxy.Service.SessionAffinity +{ + /// + /// Handles failures caused by a missing for an affinizied request. + /// + internal interface IMissingDestinationHandler + { + /// + /// A unique identifier for this missing destionation handler implementation. This will be referenced from config. + /// + public string Name { get; } + + /// + /// Handles destination affinitization failure when no was found for the given request's affinity key. + /// + /// Current request's context. + /// Request's affinity key. + /// s available for the request. + /// List of chosen to be affinitized to the request. + public IReadOnlyList Handle(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, object affinityKey, IReadOnlyList availableDestinations); + } +} diff --git a/src/ReverseProxy/Service/SessionAffinity/PickRandomMissingDestinationHandler.cs b/src/ReverseProxy/Service/SessionAffinity/PickRandomMissingDestinationHandler.cs new file mode 100644 index 000000000..ec42ea2c3 --- /dev/null +++ b/src/ReverseProxy/Service/SessionAffinity/PickRandomMissingDestinationHandler.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using Microsoft.AspNetCore.Http; +using Microsoft.ReverseProxy.RuntimeModel; + +namespace Microsoft.ReverseProxy.Service.SessionAffinity +{ + internal class PickRandomMissingDestinationHandler : IMissingDestinationHandler + { + private readonly Random _random = new Random(); + + public string Name => "PickRandom"; + + public IReadOnlyList Handle(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, object affinityKey, IReadOnlyList availableDestinations) + { + if (availableDestinations.Count == 0) + { + return availableDestinations; + } + + var index = _random.Next(availableDestinations.Count); + return new[] { availableDestinations[index] }; + } + } +} diff --git a/src/ReverseProxy/Service/SessionAffinity/ReturnErrorMissingDestinationHandler.cs b/src/ReverseProxy/Service/SessionAffinity/ReturnErrorMissingDestinationHandler.cs new file mode 100644 index 000000000..672cfaf42 --- /dev/null +++ b/src/ReverseProxy/Service/SessionAffinity/ReturnErrorMissingDestinationHandler.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Collections.Generic; +using Microsoft.AspNetCore.Http; +using Microsoft.ReverseProxy.RuntimeModel; + +namespace Microsoft.ReverseProxy.Service.SessionAffinity +{ + internal class ReturnErrorMissingDestinationHandler : IMissingDestinationHandler + { + public string Name => "ReturnError"; + + public IReadOnlyList Handle(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, object affinityKey, IReadOnlyList availableDestinations) + { + return new DestinationInfo[0]; + } + } +} diff --git a/src/ReverseProxy/Service/SessionAffinity/SessionAffinityMiddlewareHelper.cs b/src/ReverseProxy/Service/SessionAffinity/SessionAffinityMiddlewareHelper.cs new file mode 100644 index 000000000..7e63707e6 --- /dev/null +++ b/src/ReverseProxy/Service/SessionAffinity/SessionAffinityMiddlewareHelper.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using Microsoft.ReverseProxy.Service.SessionAffinity; + +namespace Microsoft.ReverseProxy.Service.SessionAffinity +{ + internal static class SessionAffinityMiddlewareHelper + { + public static IDictionary ToDictionaryById(this IEnumerable services, Func idSelector) + { + if (services == null) + { + throw new ArgumentNullException(nameof(services)); + } + + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var service in services) + { + if (!result.TryAdd(idSelector(service), service)) + { + throw new ArgumentException(nameof(services), $"More than one {nameof(T)} found with the same identifier."); + } + } + + return result; + } + + public static IDictionary ToProviderDictionary(this IEnumerable sessionAffinityProviders) + { + return ToDictionaryById(sessionAffinityProviders, p => p.Mode); + } + + public static IDictionary ToHandlerDictionary(this IEnumerable missingDestinationHandlers) + { + return ToDictionaryById(missingDestinationHandlers, p => p.Name); + } + + public static T GetRequiredServiceById(this IDictionary services, string id) + { + if (!services.TryGetValue(id, out var result)) + { + throw new ArgumentException(nameof(id), $"No {nameof(T)} was found for the id {id}."); + } + return result; + } + } +} From a81201529b3686dd1a13fd58f3176598437b2e04 Mon Sep 17 00:00:00 2001 From: Alexander Nikolaev Date: Tue, 26 May 2020 16:07:32 +0200 Subject: [PATCH 06/30] - Logging added to BaseAffinityProvider - AffinitizeRequestMiddleware recreates the destination list only if there are still multiple destinations available - AddDataProtection method removed from proxy's API. DataProtection is set up in the ReverseProxy.Sample --- samples/ReverseProxy.Sample/Startup.cs | 1 + .../IReverseProxyBuilderExtensions.cs | 12 ---- ...ReverseProxyServiceCollectionExtensions.cs | 1 - src/ReverseProxy/EventIds.cs | 6 +- .../Middleware/AffinitizeRequestMiddleware.cs | 46 +++++++++---- .../AffinitizedDestinationLookupMiddleware.cs | 9 ++- .../BaseSessionAffinityProvider.cs | 64 ++++++++++++++++--- .../CookieSessionAffinityProvider.cs | 6 +- .../CustomHeaderSessionAffinityProvider.cs | 8 ++- .../ISessionAffinityProvider.cs | 3 +- 10 files changed, 110 insertions(+), 46 deletions(-) diff --git a/samples/ReverseProxy.Sample/Startup.cs b/samples/ReverseProxy.Sample/Startup.cs index 42f9aa178..e5eb97b56 100644 --- a/samples/ReverseProxy.Sample/Startup.cs +++ b/samples/ReverseProxy.Sample/Startup.cs @@ -29,6 +29,7 @@ public Startup(IConfiguration configuration) public void ConfigureServices(IServiceCollection services) { services.AddControllers(); + services.AddDataProtection(); services.AddReverseProxy() .LoadFromConfig(_configuration.GetSection("ReverseProxy")) .AddProxyConfigFilter(); diff --git a/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs b/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs index c6242713f..fe3c4c116 100644 --- a/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs +++ b/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs @@ -101,17 +101,5 @@ public static IReverseProxyBuilder AddSessionAffinityProvider(this IReverseProxy return builder; } - - public static IReverseProxyBuilder AddDataProtection(this IReverseProxyBuilder builder) - { - builder.Services.AddDataProtection(); - return builder; - } - - public static IReverseProxyBuilder AddDataProtection(this IReverseProxyBuilder builder, Action setupAction) - { - builder.Services.AddDataProtection(setupAction); - return builder; - } } } diff --git a/src/ReverseProxy/Configuration/DependencyInjection/ReverseProxyServiceCollectionExtensions.cs b/src/ReverseProxy/Configuration/DependencyInjection/ReverseProxyServiceCollectionExtensions.cs index d99b578be..a4462253d 100644 --- a/src/ReverseProxy/Configuration/DependencyInjection/ReverseProxyServiceCollectionExtensions.cs +++ b/src/ReverseProxy/Configuration/DependencyInjection/ReverseProxyServiceCollectionExtensions.cs @@ -29,7 +29,6 @@ public static IReverseProxyBuilder AddReverseProxy(this IServiceCollection servi .AddRuntimeStateManagers() .AddConfigManager() .AddDynamicEndpointDataSource() - .AddDataProtection() .AddSessionAffinityProvider() .AddProxy() .AddBackgroundWorkers(); diff --git a/src/ReverseProxy/EventIds.cs b/src/ReverseProxy/EventIds.cs index 49bab7ca8..2999c8baa 100644 --- a/src/ReverseProxy/EventIds.cs +++ b/src/ReverseProxy/EventIds.cs @@ -41,6 +41,10 @@ internal static class EventIds public static readonly EventId OperationEnded = new EventId(32, "OperationEnded"); public static readonly EventId OperationFailed = new EventId(33, "OperationFailed"); public static readonly EventId AffinitizedDestinationIsNotFound = new EventId(34, "AffinitizedDestinationIsNotFound"); - public static readonly EventId AttemptToAffinitizeMultipleDestinations = new EventId(35, "AttemptToAffinitizeMultipleDestinations"); + public static readonly EventId MultipleDestinationsOnBackendToEstablishRequestAffinity = new EventId(35, "MultipleDestinationsOnBackendToEstablishRequestAffinity"); + public static readonly EventId AffinityCannotBeEstablishedBecauseNoDestinationsFoundOnBackend = new EventId(36, "AffinityCannotBeEstablishedBecauseNoDestinationsFoundOnBackend"); + public static readonly EventId RequestAffinityToDestinationCannotBeEstablishedBecauseAffinitizationDisabled = new EventId(37, "RequestAffinityToDestinationCannotBeEstablishedBecauseAffinitizationDisabled"); + public static readonly EventId RequestAffinityKeyAlreadyPresentInContext = new EventId(38, "RequestAffinityKeyAlreadyPresentInContext"); + public static readonly EventId NoDestinationOnBackendToEstablishRequestAffinity = new EventId(39, "NoDestinationOnBackendToEstablishRequestAffinity"); } } diff --git a/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs b/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs index f19f0facb..7263f933e 100644 --- a/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs +++ b/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs @@ -43,37 +43,55 @@ public Task Invoke(HttpContext context) if (options.Enabled) { var destinationsFeature = context.GetRequiredDestinationFeature(); - var destination = _operationLogger.Execute("ReverseProxy.AffinitizeRequest", () => AffinitizeRequest(context, backend, options, destinationsFeature.Destinations)); - destinationsFeature.Destinations = new[] { destination }; + var candidateDestinations = destinationsFeature.Destinations; + if (candidateDestinations.Count == 0) + { + Log.NoDestinationOnBackendToEstablishRequestAffinity(_logger, backend.BackendId); + context.Response.StatusCode = 503; + return Task.CompletedTask; + } + var destinations = _operationLogger.Execute("ReverseProxy.AffinitizeRequest", () => AffinitizeRequest(context, backend, options, candidateDestinations)); + destinationsFeature.Destinations = destinations; } return _next(context); } - private DestinationInfo AffinitizeRequest(HttpContext context, BackendInfo backend, BackendConfig.BackendSessionAffinityOptions options, IReadOnlyList destinations) + private IReadOnlyList AffinitizeRequest(HttpContext context, BackendInfo backend, BackendConfig.BackendSessionAffinityOptions options, IReadOnlyList destinations) { - var destination = destinations[0]; - if (destinations.Count > 1) + var result = destinations; + if (result.Count > 1) { - Log.AttemptToAffinitizeMultipleDestinations(_logger, backend.BackendId); - destination = destinations[_random.Next(destinations.Count)]; // It's assumed that all of them match to the request's affinity key. + Log.MultipleDestinationsOnBackendToEstablishRequestAffinity(_logger, backend.BackendId); + var singleDestination = destinations[_random.Next(destinations.Count)]; // It's assumed that all of them match to the request's affinity key. + result = new[] { singleDestination }; } var currentProvider = _sessionAffinityProviders.GetRequiredServiceById(options.Mode); - currentProvider.AffinitizeRequest(context, options, destination); - return destination; + currentProvider.AffinitizeRequest(context, options, result[0]); + return result; } private static class Log { - private static readonly Action _attemptToAffinitizeMultipleDestinations = LoggerMessage.Define( + private static readonly Action _multipleDestinationsOnBackendToEstablishRequestAffinity = LoggerMessage.Define( LogLevel.Warning, - EventIds.AttemptToAffinitizeMultipleDestinations, - "The request still has multiple destinations on the backend {backendId} to choose from when establishing affinity, load balancing may not be properly configured. A random destination will be used."); + EventIds.MultipleDestinationsOnBackendToEstablishRequestAffinity, + "The request still has multiple destinations on the backend `{backendId}` to choose from when establishing affinity, load balancing may not be properly configured. A random destination will be used."); - public static void AttemptToAffinitizeMultipleDestinations(ILogger logger, string backendId) + private static readonly Action _noDestinationOnBackendToEstablishRequestAffinity = LoggerMessage.Define( + LogLevel.Error, + EventIds.NoDestinationOnBackendToEstablishRequestAffinity, + "The request doesn't have any destinations on the backend `{backendId}` to choose from when establishing affinity, load balancing may not be properly configured."); + + public static void MultipleDestinationsOnBackendToEstablishRequestAffinity(ILogger logger, string backendId) + { + _multipleDestinationsOnBackendToEstablishRequestAffinity(logger, backendId, null); + } + + public static void NoDestinationOnBackendToEstablishRequestAffinity(ILogger logger, string backendId) { - _attemptToAffinitizeMultipleDestinations(logger, backendId, null); + _noDestinationOnBackendToEstablishRequestAffinity(logger, backendId, null); } } } diff --git a/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs b/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs index 3f2f18cd4..230301cc7 100644 --- a/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs +++ b/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs @@ -40,14 +40,13 @@ public Task Invoke(HttpContext context) var destinationsFeature = context.GetRequiredDestinationFeature(); var destinations = destinationsFeature.Destinations; - var options = backend.Config.Value?.SessionAffinityOptions - ?? new BackendConfig.BackendSessionAffinityOptions(false, default, default, default); + var options = backend.Config.Value?.SessionAffinityOptions ?? default; if (options.Enabled) { var affinitizedDestinations = _operationLogger.Execute( "ReverseProxy.FindAffinitizedDestinations", - () => FindAffinitizedDestinations(context, destinations, options)); + () => FindAffinitizedDestinations(context, destinations, backend, options)); if (affinitizedDestinations.DestinationsFound) { if (affinitizedDestinations.Result.Destinations.Count > 0) @@ -67,10 +66,10 @@ public Task Invoke(HttpContext context) return _next(context); } - private (bool DestinationsFound, AffinityResult Result) FindAffinitizedDestinations(HttpContext context, IReadOnlyList destinations, BackendConfig.BackendSessionAffinityOptions options) + private (bool DestinationsFound, AffinityResult Result) FindAffinitizedDestinations(HttpContext context, IReadOnlyList destinations, BackendInfo backend, BackendConfig.BackendSessionAffinityOptions options) { var currentProvider = _sessionAffinityProviders.GetRequiredServiceById(options.Mode); - var destinationsFound = currentProvider.TryFindAffinitizedDestinations(context, destinations, options, out var affinityResult); + var destinationsFound = currentProvider.TryFindAffinitizedDestinations(context, destinations, backend, options, out var affinityResult); return (destinationsFound, affinityResult); } diff --git a/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs index b314ad87e..2b5970c4b 100644 --- a/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using Microsoft.AspNetCore.DataProtection; using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; using Microsoft.ReverseProxy.RuntimeModel; namespace Microsoft.ReverseProxy.Service.SessionAffinity @@ -14,11 +15,13 @@ internal abstract class BaseSessionAffinityProvider : ISessionAffinityProvide protected static readonly object AffinityKeyId = new object(); protected readonly IDataProtector DataProtector; protected readonly IDictionary MissingDestinationHandlers; + protected readonly ILogger Logger; - protected BaseSessionAffinityProvider(IDataProtectionProvider dataProtectionProvider, IEnumerable missingDestinationHandlers) + protected BaseSessionAffinityProvider(IDataProtectionProvider dataProtectionProvider, IEnumerable missingDestinationHandlers, ILogger logger) { DataProtector = dataProtectionProvider?.CreateProtector(GetType().FullName) ?? throw new ArgumentNullException(nameof(dataProtectionProvider)); MissingDestinationHandlers = missingDestinationHandlers?.ToHandlerDictionary() ?? throw new ArgumentNullException(nameof(missingDestinationHandlers)); + Logger = logger ?? throw new ArgumentNullException(nameof(logger)); } public abstract string Mode { get; } @@ -27,6 +30,7 @@ public virtual void AffinitizeRequest(HttpContext context, BackendConfig.Backend { if (!options.Enabled) { + Log.RequestAffinityToDestinationCannotBeEstablishedBecauseAffinitizationDisabled(Logger, destination.DestinationId); return; } @@ -38,10 +42,17 @@ public virtual void AffinitizeRequest(HttpContext context, BackendConfig.Backend SetAffinityKey(context, options, (T)affinityKey); } - public virtual bool TryFindAffinitizedDestinations(HttpContext context, IReadOnlyList destinations, BackendConfig.BackendSessionAffinityOptions options, out AffinityResult affinityResult) + public virtual bool TryFindAffinitizedDestinations(HttpContext context, IReadOnlyList destinations, BackendInfo backend, BackendConfig.BackendSessionAffinityOptions options, out AffinityResult affinityResult) { - if (!options.Enabled || destinations.Count == 0) + if (!options.Enabled) + { + affinityResult = default; + return false; + } + + if (destinations.Count == 0) { + Log.AffinityCannotBeEstablishedBecauseNoDestinationsFound(Logger, backend.BackendId); affinityResult = default; return false; } @@ -54,19 +65,23 @@ public virtual bool TryFindAffinitizedDestinations(HttpContext context, IReadOnl return false; } - context.Items.Add(AffinityKeyId, requestAffinityKey); + if (!context.Items.TryAdd(AffinityKeyId, requestAffinityKey)) + { + Log.RequestAffinityKeyAlreadyPresentInContext(Logger, backend.BackendId); + throw new InvalidOperationException("Request affinitization failed."); + } - // It's allowed to affinitize a request to a pool of destinations so as to enable load-balancing among them - var matchingDestinations = new List(); + var matchingDestinations = new DestinationInfo[1]; for (var i = 0; i < destinations.Count; i++) { if (requestAffinityKey.Equals(GetDestinationAffinityKey(destinations[i]))) { - matchingDestinations.Add(destinations[i]); + matchingDestinations[0] = destinations[i]; // It's allowed to affinitize a request to a pool of destinations so as to enable load-balancing among them. + break; // However, we currently stop after the first match found to avoid performance degradation. } } - if (matchingDestinations.Count == 0) + if (matchingDestinations.Length == 0) { var failureHandler = MissingDestinationHandlers[options.MissingDestinationHandler]; var newAffinitizedDestinations = failureHandler.Handle(context, options, requestAffinityKey, destinations); @@ -95,5 +110,38 @@ protected virtual string GetSettingValue(string key, BackendConfig.BackendSessio protected abstract T GetRequestAffinityKey(HttpContext context, BackendConfig.BackendSessionAffinityOptions options); protected abstract void SetAffinityKey(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, T unencryptedKey); + + private static class Log + { + private static readonly Action _affinityCannotBeEstablishedBecauseNoDestinationsFound = LoggerMessage.Define( + LogLevel.Warning, + EventIds.AffinityCannotBeEstablishedBecauseNoDestinationsFoundOnBackend, + "The request affinity cannot be established because no destinations are found on backend `{backendId}`."); + + private static readonly Action _requestAffinityToDestinationCannotBeEstablishedBecauseAffinitizationDisabled = LoggerMessage.Define( + LogLevel.Warning, + EventIds.RequestAffinityToDestinationCannotBeEstablishedBecauseAffinitizationDisabled, + "The request affinity to destination `{destinationId}` cannot be established because affinitization is disabled for the backend."); + + private static readonly Action _requestAffinityKeyAlreadyPresentInContext = LoggerMessage.Define( + LogLevel.Error, + EventIds.RequestAffinityKeyAlreadyPresentInContext, + "The request affinity key is already present in HttpContext. Affinization failed for backend `{backendId}`."); + + public static void AffinityCannotBeEstablishedBecauseNoDestinationsFound(ILogger logger, string backendId) + { + _affinityCannotBeEstablishedBecauseNoDestinationsFound(logger, backendId, null); + } + + public static void RequestAffinityToDestinationCannotBeEstablishedBecauseAffinitizationDisabled(ILogger logger, string destinationId) + { + _requestAffinityToDestinationCannotBeEstablishedBecauseAffinitizationDisabled(logger, destinationId, null); + } + + public static void RequestAffinityKeyAlreadyPresentInContext(ILogger logger, string backendId) + { + _requestAffinityKeyAlreadyPresentInContext(logger, backendId, null); + } + } } } diff --git a/src/ReverseProxy/Service/SessionAffinity/CookieSessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/CookieSessionAffinityProvider.cs index 432bda715..0f2493f8c 100644 --- a/src/ReverseProxy/Service/SessionAffinity/CookieSessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/CookieSessionAffinityProvider.cs @@ -8,6 +8,7 @@ using Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract; using Microsoft.ReverseProxy.RuntimeModel; using System.Collections.Generic; +using Microsoft.Extensions.Logging; namespace Microsoft.ReverseProxy.Service.SessionAffinity { @@ -18,8 +19,9 @@ internal class CookieSessionAffinityProvider : BaseSessionAffinityProvider providerOptions, IDataProtectionProvider dataProtectionProvider, - IEnumerable missingDestinationHandlers) - : base(dataProtectionProvider, missingDestinationHandlers) + IEnumerable missingDestinationHandlers, + ILogger logger) + : base(dataProtectionProvider, missingDestinationHandlers, logger) { _providerOptions = providerOptions?.Value ?? throw new ArgumentNullException(nameof(providerOptions)); } diff --git a/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs index fba142340..da8ae0d86 100644 --- a/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using Microsoft.AspNetCore.DataProtection; using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Primitives; using Microsoft.ReverseProxy.RuntimeModel; @@ -13,8 +14,11 @@ internal class CustomHeaderSessionAffinityProvider : BaseSessionAffinityProvider { private const string CustomHeaderNameKey = "CustomHeaderName"; - public CustomHeaderSessionAffinityProvider(IDataProtectionProvider dataProtectionProvider, IEnumerable missingDestinationHandlers) - : base(dataProtectionProvider, missingDestinationHandlers) + public CustomHeaderSessionAffinityProvider( + IDataProtectionProvider dataProtectionProvider, + IEnumerable missingDestinationHandlers, + ILogger logger) + : base(dataProtectionProvider, missingDestinationHandlers, logger) {} public override string Mode => "CustomHeader"; diff --git a/src/ReverseProxy/Service/SessionAffinity/ISessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/ISessionAffinityProvider.cs index 97fc1298c..df6bd7c97 100644 --- a/src/ReverseProxy/Service/SessionAffinity/ISessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/ISessionAffinityProvider.cs @@ -23,10 +23,11 @@ internal interface ISessionAffinityProvider /// /// Current request's context. /// s available for the request. + /// Target backend. /// Affinity options. /// Affinitized s found for the request. /// if affinitized s were successfully found, otherwise . - public bool TryFindAffinitizedDestinations(HttpContext context, IReadOnlyList destinations, BackendConfig.BackendSessionAffinityOptions options, out AffinityResult affinityResult); + public bool TryFindAffinitizedDestinations(HttpContext context, IReadOnlyList destinations, BackendInfo backend, BackendConfig.BackendSessionAffinityOptions options, out AffinityResult affinityResult); /// /// Affinitize the current request to the given by setting the affinity key extracted from . From edf3ae724870b91630914ab580fff7a78fe76622 Mon Sep 17 00:00:00 2001 From: Alexander Nikolaev Date: Tue, 26 May 2020 16:09:26 +0200 Subject: [PATCH 07/30] AffinityResult.Destinations converted to property --- src/ReverseProxy/Service/SessionAffinity/AffinityResult.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ReverseProxy/Service/SessionAffinity/AffinityResult.cs b/src/ReverseProxy/Service/SessionAffinity/AffinityResult.cs index 236522965..42e0c2d1f 100644 --- a/src/ReverseProxy/Service/SessionAffinity/AffinityResult.cs +++ b/src/ReverseProxy/Service/SessionAffinity/AffinityResult.cs @@ -8,7 +8,7 @@ namespace Microsoft.ReverseProxy.Service.SessionAffinity { public readonly struct AffinityResult { - public readonly IReadOnlyList Destinations; + public IReadOnlyList Destinations { get; } public AffinityResult(IReadOnlyList destinations) { From ad428d2348d3eb068c163cfd8fd787da9fce4bf9 Mon Sep 17 00:00:00 2001 From: Alexander Nikolaev Date: Tue, 26 May 2020 16:23:12 +0200 Subject: [PATCH 08/30] Redundant Dictionary ctor removed --- .../Middleware/AffinitizedDestinationLookupMiddleware.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs b/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs index 230301cc7..7053d3fda 100644 --- a/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs +++ b/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs @@ -18,7 +18,7 @@ namespace Microsoft.ReverseProxy.Middleware internal class AffinitizedDestinationLookupMiddleware { private readonly RequestDelegate _next; - private readonly IDictionary _sessionAffinityProviders = new Dictionary(); + private readonly IDictionary _sessionAffinityProviders; private readonly IOperationLogger _operationLogger; private readonly ILogger _logger; From ee671540a0531d49a6301b78022a50ef98c33b59 Mon Sep 17 00:00:00 2001 From: Alexander Nikolaev Date: Tue, 26 May 2020 17:45:05 +0200 Subject: [PATCH 09/30] - Session affinity default options - BaseSessionAffinityProviderTest --- .../Contract/SessionAffinityDefaultOptions.cs | 34 +++++++++++++++ .../Service/Config/ConfigErrors.cs | 2 - .../Service/Config/DynamicConfigBuilder.cs | 25 +++++++---- .../BaseSesstionAffinityProviderTest.cs | 41 +++++++++++++++++++ 4 files changed, 91 insertions(+), 11 deletions(-) create mode 100644 src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityDefaultOptions.cs create mode 100644 test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTest.cs diff --git a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityDefaultOptions.cs b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityDefaultOptions.cs new file mode 100644 index 000000000..f627c3f58 --- /dev/null +++ b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityDefaultOptions.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; + +namespace Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract +{ + /// + /// Sets default values for session affinity configuration settings. + /// + public class SessionAffinityDefaultOptions + { + private string _defaultMode = "Cookie"; + private string _defaultMissingDestinationHandler = "ReturnError"; + + /// + /// Default session affinity mode to be used when none is specified for a backend. + /// + public string DefaultMode + { + get => _defaultMode; + set => _defaultMode = value ?? throw new ArgumentNullException(nameof(value)); + } + + /// + /// Default strategy handling missing destination for an affinitized request. + /// + public string MissingDestinationHandler + { + get => _defaultMissingDestinationHandler; + set => _defaultMissingDestinationHandler = value ?? throw new ArgumentNullException(nameof(value)); + } + } +} diff --git a/src/ReverseProxy/Service/Config/ConfigErrors.cs b/src/ReverseProxy/Service/Config/ConfigErrors.cs index 241e12039..304e25d74 100644 --- a/src/ReverseProxy/Service/Config/ConfigErrors.cs +++ b/src/ReverseProxy/Service/Config/ConfigErrors.cs @@ -23,9 +23,7 @@ internal static class ConfigErrors internal const string ParsedRouteRuleInvalidMatcher = "ParsedRoute_RuleInvalidMatcher"; internal const string ConfigBuilderBackendIdMismatch = "ConfigBuilder_BackendIdMismatch"; - internal const string ConfigBuilderBackendSessionAffinityModeIsNull = "ConfigBuilder_BackendSessionAffinityModeIsNull"; internal const string ConfigBuilderBackendNoProviderFoundForSessionAffinityMode = "ConfigBuilder_BackendNoProviderFoundForSessionAffinityMode"; - internal const string ConfigBuilderBackendMissingDestinationHandlerIsNull = "ConfigBuilder_MissingDestinationHandlerIsNull"; internal const string ConfigBuilderBackendNoMissingDestinationHandlerFoundForSpecifiedName = "ConfigBuilder_NoMissingDestinationHandlerFoundForSpecifiedName"; internal const string ConfigBuilderBackendException = "ConfigBuilder_BackendException"; internal const string ConfigBuilderRouteException = "ConfigBuilder_RouteException"; diff --git a/src/ReverseProxy/Service/Config/DynamicConfigBuilder.cs b/src/ReverseProxy/Service/Config/DynamicConfigBuilder.cs index 8aaed8e05..0be79355f 100644 --- a/src/ReverseProxy/Service/Config/DynamicConfigBuilder.cs +++ b/src/ReverseProxy/Service/Config/DynamicConfigBuilder.cs @@ -5,7 +5,9 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.Options; using Microsoft.ReverseProxy.Abstractions; +using Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract; using Microsoft.ReverseProxy.ConfigModel; using Microsoft.ReverseProxy.Service.SessionAffinity; using Microsoft.ReverseProxy.Utilities; @@ -20,6 +22,7 @@ internal class DynamicConfigBuilder : IDynamicConfigBuilder private readonly IRouteValidator _parsedRouteValidator; private readonly IDictionary _sessionAffinityProviders; private readonly IDictionary _missingDestionationHandlers; + private readonly SessionAffinityDefaultOptions _sessionAffinityDefaultOptions; public DynamicConfigBuilder( IEnumerable filters, @@ -27,7 +30,8 @@ public DynamicConfigBuilder( IRoutesRepo routesRepo, IRouteValidator parsedRouteValidator, IEnumerable sessionAffinityProviders, - IEnumerable missingDestinationHandlers) + IEnumerable missingDestinationHandlers, + IOptions sessionAffinityDefaultOptions) { Contracts.CheckValue(filters, nameof(filters)); Contracts.CheckValue(backendsRepo, nameof(backendsRepo)); @@ -41,6 +45,7 @@ public DynamicConfigBuilder( _parsedRouteValidator = parsedRouteValidator; _sessionAffinityProviders = sessionAffinityProviders.ToProviderDictionary(); _missingDestionationHandlers = missingDestinationHandlers.ToHandlerDictionary(); + _sessionAffinityDefaultOptions = sessionAffinityDefaultOptions?.Value ?? throw new ArgumentNullException(nameof(sessionAffinityDefaultOptions)); } public async Task> BuildConfigAsync(IConfigErrorReporter errorReporter, CancellationToken cancellation) @@ -100,22 +105,24 @@ private void ValidateSessionAffinity(IConfigErrorReporter errorReporter, string return; } - var affinityMode = backend.SessionAffinity.Mode; - if (affinityMode == null) + if (backend.SessionAffinity.Mode == null) { - errorReporter.ReportError(ConfigErrors.ConfigBuilderBackendSessionAffinityModeIsNull, id, $"The session affinity mode is null for the backend {backend.Id}."); + backend.SessionAffinity.Mode = _sessionAffinityDefaultOptions.DefaultMode; } - else if (!_sessionAffinityProviders.ContainsKey(affinityMode)) + + var affinityMode = backend.SessionAffinity.Mode; + if (!_sessionAffinityProviders.ContainsKey(affinityMode)) { errorReporter.ReportError(ConfigErrors.ConfigBuilderBackendNoProviderFoundForSessionAffinityMode, id, $"No matching {nameof(ISessionAffinityProvider)} found for the session affinity mode {affinityMode} set on the backend {backend.Id}."); } - var missingDestinationHandler = backend.SessionAffinity.MissingDestinationHandler; - if (missingDestinationHandler == null) + if (backend.SessionAffinity.MissingDestinationHandler == null) { - errorReporter.ReportError(ConfigErrors.ConfigBuilderBackendMissingDestinationHandlerIsNull, id, $"The missing affinitizated destination handler name is null for the backend {backend.Id}."); + backend.SessionAffinity.MissingDestinationHandler = _sessionAffinityDefaultOptions.MissingDestinationHandler; } - else if (!_missingDestionationHandlers.ContainsKey(missingDestinationHandler)) + + var missingDestinationHandler = backend.SessionAffinity.MissingDestinationHandler; + if (!_missingDestionationHandlers.ContainsKey(missingDestinationHandler)) { errorReporter.ReportError(ConfigErrors.ConfigBuilderBackendNoMissingDestinationHandlerFoundForSpecifiedName, id, $"No matching {nameof(IMissingDestinationHandler)} found for the missing affinitizated destination handler name {missingDestinationHandler} set on the backend {backend.Id}."); } diff --git a/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTest.cs b/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTest.cs new file mode 100644 index 000000000..43f385260 --- /dev/null +++ b/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTest.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Collections.Generic; +using Microsoft.AspNetCore.DataProtection; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using Microsoft.ReverseProxy.RuntimeModel; +using Tests.Common; + +namespace Microsoft.ReverseProxy.Service.SessionAffinity +{ + public class BaseSesstionAffinityProviderTest : TestAutoMockBase + { + private class SessionAffinityProviderStub : BaseSessionAffinityProvider + { + public static readonly string AffinityKeyItemName = "StubAffinityKey"; + + public SessionAffinityProviderStub(IDataProtectionProvider dataProtectionProvider, IEnumerable missingDestinationHandlers, ILogger logger) + : base(dataProtectionProvider, missingDestinationHandlers, logger) + {} + + public override string Mode => "Stub"; + + protected override string GetDestinationAffinityKey(DestinationInfo destination) + { + return destination.DestinationId; + } + + protected override string GetRequestAffinityKey(HttpContext context, BackendConfig.BackendSessionAffinityOptions options) + { + return (string)context.Items[AffinityKeyItemName]; + } + + protected override void SetAffinityKey(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, string unencryptedKey) + { + context.Items[AffinityKeyItemName] = unencryptedKey; + } + } + } +} From db9afae25617ac0c20ae141fbbde03ab84e9f858 Mon Sep 17 00:00:00 2001 From: Alexander Nikolaev Date: Wed, 27 May 2020 17:21:24 +0200 Subject: [PATCH 10/30] - ISessionAffinityProvider and BackendConfig are made public - AddSessionAffinity method can enable the default session affinity for all backends - Names of built-in SessionAffinity services are put into SessionAffinityBuiltIns static type - Affinity cookie protection is fixed --- samples/ReverseProxy.Sample/appsettings.json | 4 - .../Contract/SessionAffinityBuiltIns.cs | 25 ++++++ .../Contract/SessionAffinityDefaultOptions.cs | 10 ++- .../IReverseProxyBuilderExtensions.cs | 7 +- ...ReverseProxyServiceCollectionExtensions.cs | 2 +- src/ReverseProxy/EventIds.cs | 5 +- .../AffinitizedDestinationLookupMiddleware.cs | 2 +- .../Service/Config/DynamicConfigBuilder.cs | 20 +++-- .../Service/RuntimeModel/BackendConfig.cs | 9 +- .../BaseSessionAffinityProvider.cs | 53 +++++------- .../CookieSessionAffinityProvider.cs | 84 +++++++++++++++++-- .../CustomHeaderSessionAffinityProvider.cs | 7 +- .../ISessionAffinityProvider.cs | 8 +- .../PickRandomMissingDestinationHandler.cs | 3 +- .../ReturnErrorMissingDestinationHandler.cs | 3 +- .../SessionAffinityMiddlewareHelper.cs | 2 +- .../BaseSesstionAffinityProviderTest.cs | 4 +- 17 files changed, 177 insertions(+), 71 deletions(-) create mode 100644 src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityBuiltIns.cs diff --git a/samples/ReverseProxy.Sample/appsettings.json b/samples/ReverseProxy.Sample/appsettings.json index 8ee0cc0d8..c42715bbb 100644 --- a/samples/ReverseProxy.Sample/appsettings.json +++ b/samples/ReverseProxy.Sample/appsettings.json @@ -23,10 +23,6 @@ "Metadata": { "CustomHealth": "false" }, - "SessionAffinity": { - "Mode": "Cookie", - "MissingDestinationHandler": "ReturnError" - }, "Destinations": { "backend1/destination1": { "Address": "https://localhost:10000/" diff --git a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityBuiltIns.cs b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityBuiltIns.cs new file mode 100644 index 000000000..8a19580c5 --- /dev/null +++ b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityBuiltIns.cs @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract +{ + /// + /// Names of built-in session affinity services. + /// + public static class SessionAffinityBuiltIns + { + public static class Modes + { + public static string Cookie => "Cookie"; + + public static string CustomHeander => "CustomHeader"; + } + + public static class MissingDestinationHandlers + { + public static string PickRandom => "PickRandom"; + + public static string ReturnError => "ReturnError"; + } + } +} diff --git a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityDefaultOptions.cs b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityDefaultOptions.cs index f627c3f58..39548cc04 100644 --- a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityDefaultOptions.cs +++ b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityDefaultOptions.cs @@ -10,8 +10,8 @@ namespace Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract /// public class SessionAffinityDefaultOptions { - private string _defaultMode = "Cookie"; - private string _defaultMissingDestinationHandler = "ReturnError"; + private string _defaultMode = SessionAffinityBuiltIns.Modes.Cookie; + private string _defaultMissingDestinationHandler = SessionAffinityBuiltIns.MissingDestinationHandlers.ReturnError; /// /// Default session affinity mode to be used when none is specified for a backend. @@ -30,5 +30,11 @@ public string MissingDestinationHandler get => _defaultMissingDestinationHandler; set => _defaultMissingDestinationHandler = value ?? throw new ArgumentNullException(nameof(value)); } + + /// + /// If set to enables session affinity for all backends using the default settings + /// which can be ovewritten by backend's configuration section. + /// + public bool EnabledForAllBackends { get; set; } } } diff --git a/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs b/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs index fe3c4c116..9607d7482 100644 --- a/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs +++ b/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs @@ -88,8 +88,13 @@ public static IReverseProxyBuilder AddBackgroundWorkers(this IReverseProxyBuilde return builder; } - public static IReverseProxyBuilder AddSessionAffinityProvider(this IReverseProxyBuilder builder) + public static IReverseProxyBuilder AddSessionAffinityProvider(this IReverseProxyBuilder builder, bool enableForAllBackends) { + if (enableForAllBackends) + { + builder.Services.AddOptions().Configure(o => o.EnabledForAllBackends = true); + } + builder.Services.TryAddEnumerable(new[] { new ServiceDescriptor(typeof(IMissingDestinationHandler), typeof(PickRandomMissingDestinationHandler), ServiceLifetime.Singleton), new ServiceDescriptor(typeof(IMissingDestinationHandler), typeof(ReturnErrorMissingDestinationHandler), ServiceLifetime.Singleton) diff --git a/src/ReverseProxy/Configuration/DependencyInjection/ReverseProxyServiceCollectionExtensions.cs b/src/ReverseProxy/Configuration/DependencyInjection/ReverseProxyServiceCollectionExtensions.cs index a4462253d..235b172d4 100644 --- a/src/ReverseProxy/Configuration/DependencyInjection/ReverseProxyServiceCollectionExtensions.cs +++ b/src/ReverseProxy/Configuration/DependencyInjection/ReverseProxyServiceCollectionExtensions.cs @@ -29,7 +29,7 @@ public static IReverseProxyBuilder AddReverseProxy(this IServiceCollection servi .AddRuntimeStateManagers() .AddConfigManager() .AddDynamicEndpointDataSource() - .AddSessionAffinityProvider() + .AddSessionAffinityProvider(true) .AddProxy() .AddBackgroundWorkers(); diff --git a/src/ReverseProxy/EventIds.cs b/src/ReverseProxy/EventIds.cs index 2999c8baa..b8dcd4208 100644 --- a/src/ReverseProxy/EventIds.cs +++ b/src/ReverseProxy/EventIds.cs @@ -44,7 +44,8 @@ internal static class EventIds public static readonly EventId MultipleDestinationsOnBackendToEstablishRequestAffinity = new EventId(35, "MultipleDestinationsOnBackendToEstablishRequestAffinity"); public static readonly EventId AffinityCannotBeEstablishedBecauseNoDestinationsFoundOnBackend = new EventId(36, "AffinityCannotBeEstablishedBecauseNoDestinationsFoundOnBackend"); public static readonly EventId RequestAffinityToDestinationCannotBeEstablishedBecauseAffinitizationDisabled = new EventId(37, "RequestAffinityToDestinationCannotBeEstablishedBecauseAffinitizationDisabled"); - public static readonly EventId RequestAffinityKeyAlreadyPresentInContext = new EventId(38, "RequestAffinityKeyAlreadyPresentInContext"); - public static readonly EventId NoDestinationOnBackendToEstablishRequestAffinity = new EventId(39, "NoDestinationOnBackendToEstablishRequestAffinity"); + public static readonly EventId NoDestinationOnBackendToEstablishRequestAffinity = new EventId(38, "NoDestinationOnBackendToEstablishRequestAffinity"); + public static readonly EventId RequestAffinityKeyCookieCannotBeDecodedFromBase64 = new EventId(39, "RequestAffinityKeyCookieCannotBeDecodedFromBase64"); + public static readonly EventId RequestAffinityKeyCookieDecryptionFailed = new EventId(40, "RequestAffinityKeyCookieDecryptionFailed"); } } diff --git a/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs b/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs index 7053d3fda..0c34d2978 100644 --- a/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs +++ b/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs @@ -69,7 +69,7 @@ public Task Invoke(HttpContext context) private (bool DestinationsFound, AffinityResult Result) FindAffinitizedDestinations(HttpContext context, IReadOnlyList destinations, BackendInfo backend, BackendConfig.BackendSessionAffinityOptions options) { var currentProvider = _sessionAffinityProviders.GetRequiredServiceById(options.Mode); - var destinationsFound = currentProvider.TryFindAffinitizedDestinations(context, destinations, backend, options, out var affinityResult); + var destinationsFound = currentProvider.TryFindAffinitizedDestinations(context, destinations, backend.BackendId, options, out var affinityResult); return (destinationsFound, affinityResult); } diff --git a/src/ReverseProxy/Service/Config/DynamicConfigBuilder.cs b/src/ReverseProxy/Service/Config/DynamicConfigBuilder.cs index 0be79355f..44c851cfe 100644 --- a/src/ReverseProxy/Service/Config/DynamicConfigBuilder.cs +++ b/src/ReverseProxy/Service/Config/DynamicConfigBuilder.cs @@ -80,13 +80,13 @@ public async Task> GetBackendsAsync(IConfigErrorRep continue; } - ValidateSessionAffinity(errorReporter, id, backend); - foreach (var filter in _filters) { await filter.ConfigureBackendAsync(backend, cancellation); } + ValidateSessionAffinity(errorReporter, id, backend); + configuredBackends[id] = backend; } catch (Exception ex) @@ -100,12 +100,20 @@ public async Task> GetBackendsAsync(IConfigErrorRep private void ValidateSessionAffinity(IConfigErrorReporter errorReporter, string id, Backend backend) { - if (backend.SessionAffinity == null) // Session affinity is disabled + if (backend.SessionAffinity == null) { - return; + if (_sessionAffinityDefaultOptions.EnabledForAllBackends) + { + backend.SessionAffinity = new SessionAffinityOptions(); + } + else + { + // Session affinity is disabled + return; + } } - if (backend.SessionAffinity.Mode == null) + if (string.IsNullOrEmpty(backend.SessionAffinity.Mode)) { backend.SessionAffinity.Mode = _sessionAffinityDefaultOptions.DefaultMode; } @@ -116,7 +124,7 @@ private void ValidateSessionAffinity(IConfigErrorReporter errorReporter, string errorReporter.ReportError(ConfigErrors.ConfigBuilderBackendNoProviderFoundForSessionAffinityMode, id, $"No matching {nameof(ISessionAffinityProvider)} found for the session affinity mode {affinityMode} set on the backend {backend.Id}."); } - if (backend.SessionAffinity.MissingDestinationHandler == null) + if (string.IsNullOrEmpty(backend.SessionAffinity.MissingDestinationHandler)) { backend.SessionAffinity.MissingDestinationHandler = _sessionAffinityDefaultOptions.MissingDestinationHandler; } diff --git a/src/ReverseProxy/Service/RuntimeModel/BackendConfig.cs b/src/ReverseProxy/Service/RuntimeModel/BackendConfig.cs index fc5835ba8..0b4d47f6c 100644 --- a/src/ReverseProxy/Service/RuntimeModel/BackendConfig.cs +++ b/src/ReverseProxy/Service/RuntimeModel/BackendConfig.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; using Microsoft.ReverseProxy.Abstractions; -using Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract; using Microsoft.ReverseProxy.Utilities; namespace Microsoft.ReverseProxy.RuntimeModel @@ -19,7 +18,7 @@ namespace Microsoft.ReverseProxy.RuntimeModel /// Instead, instances of are replaced /// in ther entirety when values need to change. /// - internal sealed class BackendConfig + public sealed class BackendConfig { public BackendConfig( BackendHealthCheckOptions healthCheckOptions, @@ -44,7 +43,7 @@ public BackendConfig( /// Struct used only to keep things organized as we add more configuration options inside of `BackendConfig`. /// Each "feature" can have its own struct. /// - internal readonly struct BackendHealthCheckOptions + public readonly struct BackendHealthCheckOptions { public BackendHealthCheckOptions(bool enabled, TimeSpan interval, TimeSpan timeout, int port, string path) { @@ -81,7 +80,7 @@ public BackendHealthCheckOptions(bool enabled, TimeSpan interval, TimeSpan timeo public string Path { get; } } - internal readonly struct BackendLoadBalancingOptions + public readonly struct BackendLoadBalancingOptions { public BackendLoadBalancingOptions(LoadBalancingMode mode) { @@ -95,7 +94,7 @@ public BackendLoadBalancingOptions(LoadBalancingMode mode) internal AtomicCounter RoundRobinState { get; } } - internal readonly struct BackendSessionAffinityOptions + public readonly struct BackendSessionAffinityOptions { public BackendSessionAffinityOptions(bool enabled, string mode, string missingDestinationHandler, IReadOnlyDictionary settings) { diff --git a/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs index 2b5970c4b..2e14e2aae 100644 --- a/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs @@ -26,7 +26,7 @@ protected BaseSessionAffinityProvider(IDataProtectionProvider dataProtectionProv public abstract string Mode { get; } - public virtual void AffinitizeRequest(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, DestinationInfo destination) + public virtual void AffinitizeRequest(HttpContext context, in BackendConfig.BackendSessionAffinityOptions options, DestinationInfo destination) { if (!options.Enabled) { @@ -42,7 +42,7 @@ public virtual void AffinitizeRequest(HttpContext context, BackendConfig.Backend SetAffinityKey(context, options, (T)affinityKey); } - public virtual bool TryFindAffinitizedDestinations(HttpContext context, IReadOnlyList destinations, BackendInfo backend, BackendConfig.BackendSessionAffinityOptions options, out AffinityResult affinityResult) + public virtual bool TryFindAffinitizedDestinations(HttpContext context, IReadOnlyList destinations, string backendId, in BackendConfig.BackendSessionAffinityOptions options, out AffinityResult affinityResult) { if (!options.Enabled) { @@ -50,13 +50,6 @@ public virtual bool TryFindAffinitizedDestinations(HttpContext context, IReadOnl return false; } - if (destinations.Count == 0) - { - Log.AffinityCannotBeEstablishedBecauseNoDestinationsFound(Logger, backend.BackendId); - affinityResult = default; - return false; - } - var requestAffinityKey = GetRequestAffinityKey(context, options); if (requestAffinityKey == null) @@ -65,23 +58,29 @@ public virtual bool TryFindAffinitizedDestinations(HttpContext context, IReadOnl return false; } - if (!context.Items.TryAdd(AffinityKeyId, requestAffinityKey)) - { - Log.RequestAffinityKeyAlreadyPresentInContext(Logger, backend.BackendId); - throw new InvalidOperationException("Request affinitization failed."); - } - var matchingDestinations = new DestinationInfo[1]; - for (var i = 0; i < destinations.Count; i++) + if (destinations.Count > 0) { - if (requestAffinityKey.Equals(GetDestinationAffinityKey(destinations[i]))) + context.Items[AffinityKeyId] = requestAffinityKey; + + for (var i = 0; i < destinations.Count; i++) { - matchingDestinations[0] = destinations[i]; // It's allowed to affinitize a request to a pool of destinations so as to enable load-balancing among them. - break; // However, we currently stop after the first match found to avoid performance degradation. + if (requestAffinityKey.Equals(GetDestinationAffinityKey(destinations[i]))) + { + // It's allowed to affinitize a request to a pool of destinations so as to enable load-balancing among them. + // However, we currently stop after the first match found to avoid performance degradation. + matchingDestinations[0] = destinations[i]; + break; + } } } + else + { + Log.AffinityCannotBeEstablishedBecauseNoDestinationsFound(Logger, backendId); + } - if (matchingDestinations.Length == 0) + // Empty destination list passed to this method is handled the same way as if no matching destinations are found. + if (matchingDestinations[0] == null) { var failureHandler = MissingDestinationHandlers[options.MissingDestinationHandler]; var newAffinitizedDestinations = failureHandler.Handle(context, options, requestAffinityKey, destinations); @@ -107,9 +106,9 @@ protected virtual string GetSettingValue(string key, BackendConfig.BackendSessio protected abstract T GetDestinationAffinityKey(DestinationInfo destination); - protected abstract T GetRequestAffinityKey(HttpContext context, BackendConfig.BackendSessionAffinityOptions options); + protected abstract T GetRequestAffinityKey(HttpContext context, in BackendConfig.BackendSessionAffinityOptions options); - protected abstract void SetAffinityKey(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, T unencryptedKey); + protected abstract void SetAffinityKey(HttpContext context, in BackendConfig.BackendSessionAffinityOptions options, T unencryptedKey); private static class Log { @@ -123,11 +122,6 @@ private static class Log EventIds.RequestAffinityToDestinationCannotBeEstablishedBecauseAffinitizationDisabled, "The request affinity to destination `{destinationId}` cannot be established because affinitization is disabled for the backend."); - private static readonly Action _requestAffinityKeyAlreadyPresentInContext = LoggerMessage.Define( - LogLevel.Error, - EventIds.RequestAffinityKeyAlreadyPresentInContext, - "The request affinity key is already present in HttpContext. Affinization failed for backend `{backendId}`."); - public static void AffinityCannotBeEstablishedBecauseNoDestinationsFound(ILogger logger, string backendId) { _affinityCannotBeEstablishedBecauseNoDestinationsFound(logger, backendId, null); @@ -137,11 +131,6 @@ public static void RequestAffinityToDestinationCannotBeEstablishedBecauseAffinit { _requestAffinityToDestinationCannotBeEstablishedBecauseAffinitizationDisabled(logger, destinationId, null); } - - public static void RequestAffinityKeyAlreadyPresentInContext(ILogger logger, string backendId) - { - _requestAffinityKeyAlreadyPresentInContext(logger, backendId, null); - } } } } diff --git a/src/ReverseProxy/Service/SessionAffinity/CookieSessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/CookieSessionAffinityProvider.cs index 0f2493f8c..965f58837 100644 --- a/src/ReverseProxy/Service/SessionAffinity/CookieSessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/CookieSessionAffinityProvider.cs @@ -9,6 +9,7 @@ using Microsoft.ReverseProxy.RuntimeModel; using System.Collections.Generic; using Microsoft.Extensions.Logging; +using System.Text; namespace Microsoft.ReverseProxy.Service.SessionAffinity { @@ -26,23 +27,96 @@ public CookieSessionAffinityProvider( _providerOptions = providerOptions?.Value ?? throw new ArgumentNullException(nameof(providerOptions)); } - public override string Mode => "Cookie"; + public override string Mode => SessionAffinityBuiltIns.Modes.Cookie; protected override string GetDestinationAffinityKey(DestinationInfo destination) { return destination.DestinationId; } - protected override string GetRequestAffinityKey(HttpContext context, BackendConfig.BackendSessionAffinityOptions options) + protected override string GetRequestAffinityKey(HttpContext context, in BackendConfig.BackendSessionAffinityOptions options) { var encryptedRequestKey = context.Request.Cookies.TryGetValue(_providerOptions.Cookie.Name, out var keyInCookie) ? keyInCookie : null; - return encryptedRequestKey != null ? DataProtector.Unprotect(encryptedRequestKey) : null; + return !string.IsNullOrEmpty(encryptedRequestKey) ? Unprotect(encryptedRequestKey) : null; } - protected override void SetAffinityKey(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, string unencryptedKey) + protected override void SetAffinityKey(HttpContext context, in BackendConfig.BackendSessionAffinityOptions options, string unencryptedKey) { var affinityCookieOptions = _providerOptions.Cookie.Build(context); - context.Response.Cookies.Append(_providerOptions.Cookie.Name, DataProtector.Protect(unencryptedKey), affinityCookieOptions); + context.Response.Cookies.Append(_providerOptions.Cookie.Name, Protect(unencryptedKey), affinityCookieOptions); + } + + private string Protect(string unencryptedKey) + { + if (string.IsNullOrEmpty(unencryptedKey)) + { + return unencryptedKey; + } + + var userData = Encoding.UTF8.GetBytes(unencryptedKey); + + var protectedData = DataProtector.Protect(userData); + return Convert.ToBase64String(protectedData).TrimEnd('='); + } + + private string Unprotect(string encryptedRequestKey) + { + try + { + var keyBytes = Convert.FromBase64String(Pad(encryptedRequestKey)); + if (keyBytes == null) + { + Log.RequestAffinityKeyCookieCannotBeDecodedFromBase64(Logger); + return null; + } + + var decryptedKeyBytes = DataProtector.Unprotect(keyBytes); + if (decryptedKeyBytes == null) + { + Log.RequestAffinityKeyCookieDecryptionFailed(Logger, null); + return null; + } + + return Encoding.UTF8.GetString(decryptedKeyBytes); + } + catch (Exception ex) + { + Log.RequestAffinityKeyCookieDecryptionFailed(Logger, ex); + return null; + } + } + + private static string Pad(string text) + { + var padding = 3 - ((text.Length + 3) % 4); + if (padding == 0) + { + return text; + } + return text + new string('=', padding); + } + + private static class Log + { + private static readonly Action _requestAffinityKeyCookieDecryptionFailed = LoggerMessage.Define( + LogLevel.Error, + EventIds.RequestAffinityKeyCookieDecryptionFailed, + "The request affinity key cookie decryption failed."); + + private static readonly Action _requestAffinityKeyCookieCannotBeDecodedFromBase64 = LoggerMessage.Define( + LogLevel.Error, + EventIds.RequestAffinityKeyCookieCannotBeDecodedFromBase64, + "The request affinity key cookie cannot be decoded from Base64 representation."); + + public static void RequestAffinityKeyCookieDecryptionFailed(ILogger logger, Exception ex) + { + _requestAffinityKeyCookieDecryptionFailed(logger, ex); + } + + public static void RequestAffinityKeyCookieCannotBeDecodedFromBase64(ILogger logger) + { + _requestAffinityKeyCookieCannotBeDecodedFromBase64(logger, null); + } } } } diff --git a/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs index da8ae0d86..9119b49d6 100644 --- a/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs @@ -6,6 +6,7 @@ using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Primitives; +using Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract; using Microsoft.ReverseProxy.RuntimeModel; namespace Microsoft.ReverseProxy.Service.SessionAffinity @@ -21,14 +22,14 @@ public CustomHeaderSessionAffinityProvider( : base(dataProtectionProvider, missingDestinationHandlers, logger) {} - public override string Mode => "CustomHeader"; + public override string Mode => SessionAffinityBuiltIns.Modes.CustomHeander; protected override string GetDestinationAffinityKey(DestinationInfo destination) { return destination.DestinationId; } - protected override string GetRequestAffinityKey(HttpContext context, BackendConfig.BackendSessionAffinityOptions options) + protected override string GetRequestAffinityKey(HttpContext context, in BackendConfig.BackendSessionAffinityOptions options) { var customHeaderName = GetSettingValue(CustomHeaderNameKey, options); var keyHeaderValues = context.Request.Headers[customHeaderName]; @@ -36,7 +37,7 @@ protected override string GetRequestAffinityKey(HttpContext context, BackendConf return encryptedRequestKey != null ? DataProtector.Unprotect(encryptedRequestKey) : null; } - protected override void SetAffinityKey(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, string unencryptedKey) + protected override void SetAffinityKey(HttpContext context, in BackendConfig.BackendSessionAffinityOptions options, string unencryptedKey) { var customHeaderName = GetSettingValue(CustomHeaderNameKey, options); context.Response.Headers.Append(customHeaderName, DataProtector.Protect(unencryptedKey)); diff --git a/src/ReverseProxy/Service/SessionAffinity/ISessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/ISessionAffinityProvider.cs index df6bd7c97..6dfdd690e 100644 --- a/src/ReverseProxy/Service/SessionAffinity/ISessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/ISessionAffinityProvider.cs @@ -11,7 +11,7 @@ namespace Microsoft.ReverseProxy.Service.SessionAffinity /// /// Provides session affinity for load-balanced backends. /// - internal interface ISessionAffinityProvider + public interface ISessionAffinityProvider { /// /// A unique identifier for this session affinity implementation. This will be referenced from config. @@ -23,11 +23,11 @@ internal interface ISessionAffinityProvider /// /// Current request's context. /// s available for the request. - /// Target backend. + /// Target backend ID. /// Affinity options. /// Affinitized s found for the request. /// if affinitized s were successfully found, otherwise . - public bool TryFindAffinitizedDestinations(HttpContext context, IReadOnlyList destinations, BackendInfo backend, BackendConfig.BackendSessionAffinityOptions options, out AffinityResult affinityResult); + public bool TryFindAffinitizedDestinations(HttpContext context, IReadOnlyList destinations, string backendId, in BackendConfig.BackendSessionAffinityOptions options, out AffinityResult affinityResult); /// /// Affinitize the current request to the given by setting the affinity key extracted from . @@ -35,6 +35,6 @@ internal interface ISessionAffinityProvider /// Current request's context. /// Affinity options. /// to which request is to be affinitized. - public void AffinitizeRequest(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, DestinationInfo destination); + public void AffinitizeRequest(HttpContext context, in BackendConfig.BackendSessionAffinityOptions options, DestinationInfo destination); } } diff --git a/src/ReverseProxy/Service/SessionAffinity/PickRandomMissingDestinationHandler.cs b/src/ReverseProxy/Service/SessionAffinity/PickRandomMissingDestinationHandler.cs index ec42ea2c3..831eece91 100644 --- a/src/ReverseProxy/Service/SessionAffinity/PickRandomMissingDestinationHandler.cs +++ b/src/ReverseProxy/Service/SessionAffinity/PickRandomMissingDestinationHandler.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using Microsoft.AspNetCore.Http; +using Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract; using Microsoft.ReverseProxy.RuntimeModel; namespace Microsoft.ReverseProxy.Service.SessionAffinity @@ -12,7 +13,7 @@ internal class PickRandomMissingDestinationHandler : IMissingDestinationHandler { private readonly Random _random = new Random(); - public string Name => "PickRandom"; + public string Name => SessionAffinityBuiltIns.MissingDestinationHandlers.PickRandom; public IReadOnlyList Handle(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, object affinityKey, IReadOnlyList availableDestinations) { diff --git a/src/ReverseProxy/Service/SessionAffinity/ReturnErrorMissingDestinationHandler.cs b/src/ReverseProxy/Service/SessionAffinity/ReturnErrorMissingDestinationHandler.cs index 672cfaf42..2caf63e0c 100644 --- a/src/ReverseProxy/Service/SessionAffinity/ReturnErrorMissingDestinationHandler.cs +++ b/src/ReverseProxy/Service/SessionAffinity/ReturnErrorMissingDestinationHandler.cs @@ -3,13 +3,14 @@ using System.Collections.Generic; using Microsoft.AspNetCore.Http; +using Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract; using Microsoft.ReverseProxy.RuntimeModel; namespace Microsoft.ReverseProxy.Service.SessionAffinity { internal class ReturnErrorMissingDestinationHandler : IMissingDestinationHandler { - public string Name => "ReturnError"; + public string Name => SessionAffinityBuiltIns.MissingDestinationHandlers.ReturnError; public IReadOnlyList Handle(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, object affinityKey, IReadOnlyList availableDestinations) { diff --git a/src/ReverseProxy/Service/SessionAffinity/SessionAffinityMiddlewareHelper.cs b/src/ReverseProxy/Service/SessionAffinity/SessionAffinityMiddlewareHelper.cs index 7e63707e6..c918e37df 100644 --- a/src/ReverseProxy/Service/SessionAffinity/SessionAffinityMiddlewareHelper.cs +++ b/src/ReverseProxy/Service/SessionAffinity/SessionAffinityMiddlewareHelper.cs @@ -16,7 +16,7 @@ public static IDictionary ToDictionaryById(this IEnumerable ser throw new ArgumentNullException(nameof(services)); } - var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + var result = new Dictionary(StringComparer.Ordinal); foreach (var service in services) { diff --git a/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTest.cs b/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTest.cs index 43f385260..5e51a9f48 100644 --- a/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTest.cs +++ b/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTest.cs @@ -27,12 +27,12 @@ protected override string GetDestinationAffinityKey(DestinationInfo destination) return destination.DestinationId; } - protected override string GetRequestAffinityKey(HttpContext context, BackendConfig.BackendSessionAffinityOptions options) + protected override string GetRequestAffinityKey(HttpContext context, in BackendConfig.BackendSessionAffinityOptions options) { return (string)context.Items[AffinityKeyItemName]; } - protected override void SetAffinityKey(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, string unencryptedKey) + protected override void SetAffinityKey(HttpContext context, in BackendConfig.BackendSessionAffinityOptions options, string unencryptedKey) { context.Items[AffinityKeyItemName] = unencryptedKey; } From ce858b86d3a0e4fdb3a408bdc7108dc1e8197d26 Mon Sep 17 00:00:00 2001 From: Alexander Nikolaev Date: Thu, 28 May 2020 16:03:57 +0200 Subject: [PATCH 11/30] - AffinitizedDestinationLookupMiddleware checks the affinity status to detect errors - Session affinity failure handling policy is invoked by the middleware and takes the full responsibility for failure handling - Default AffinityFailurePolicy is Redistribute - Session affinity is disabled by default --- samples/ReverseProxy.Sample/Startup.cs | 1 + samples/ReverseProxy.Sample/appsettings.json | 3 + .../Contract/SessionAffinityBuiltIns.cs | 6 +- .../Contract/SessionAffinityDefaultOptions.cs | 10 +- .../Contract/SessionAffinityOptions.cs | 4 +- .../IReverseProxyBuilderExtensions.cs | 16 +-- ...ReverseProxyServiceCollectionExtensions.cs | 2 +- src/ReverseProxy/EventIds.cs | 2 +- .../AffinitizedDestinationLookupMiddleware.cs | 69 ++++++----- .../Service/Config/ConfigErrors.cs | 2 +- .../Service/Config/DynamicConfigBuilder.cs | 18 +-- .../Management/ReverseProxyConfigManager.cs | 2 +- .../Service/RuntimeModel/BackendConfig.cs | 6 +- .../Service/SessionAffinity/AffinityResult.cs | 8 +- .../Service/SessionAffinity/AffinityStatus.cs | 17 +++ .../BaseSessionAffinityProvider.cs | 110 ++++++++++++++---- .../CookieSessionAffinityProvider.cs | 82 +------------ .../CustomHeaderSessionAffinityProvider.cs | 10 +- .../SessionAffinity/IAffinityFailurePolicy.cs | 34 ++++++ .../IMissingDestinationHandler.cs | 29 ----- .../ISessionAffinityProvider.cs | 7 +- .../PickRandomMissingDestinationHandler.cs | 21 ++-- .../ReturnErrorMissingDestinationHandler.cs | 11 +- .../SessionAffinityMiddlewareHelper.cs | 4 +- .../BaseSesstionAffinityProviderTest.cs | 9 +- 25 files changed, 256 insertions(+), 227 deletions(-) create mode 100644 src/ReverseProxy/Service/SessionAffinity/AffinityStatus.cs create mode 100644 src/ReverseProxy/Service/SessionAffinity/IAffinityFailurePolicy.cs delete mode 100644 src/ReverseProxy/Service/SessionAffinity/IMissingDestinationHandler.cs diff --git a/samples/ReverseProxy.Sample/Startup.cs b/samples/ReverseProxy.Sample/Startup.cs index e5eb97b56..85eb30cfb 100644 --- a/samples/ReverseProxy.Sample/Startup.cs +++ b/samples/ReverseProxy.Sample/Startup.cs @@ -4,6 +4,7 @@ using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract; using Microsoft.ReverseProxy.Middleware; namespace Microsoft.ReverseProxy.Sample diff --git a/samples/ReverseProxy.Sample/appsettings.json b/samples/ReverseProxy.Sample/appsettings.json index c42715bbb..d2af0dc1f 100644 --- a/samples/ReverseProxy.Sample/appsettings.json +++ b/samples/ReverseProxy.Sample/appsettings.json @@ -20,6 +20,9 @@ "LoadBalancing": { "Mode": "Random" }, + "SessionAffinity": { + "Mode": "Cookie" + }, "Metadata": { "CustomHealth": "false" }, diff --git a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityBuiltIns.cs b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityBuiltIns.cs index 8a19580c5..4df825862 100644 --- a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityBuiltIns.cs +++ b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityBuiltIns.cs @@ -15,11 +15,11 @@ public static class Modes public static string CustomHeander => "CustomHeader"; } - public static class MissingDestinationHandlers + public static class AffinityFailurePolicies { - public static string PickRandom => "PickRandom"; + public static string Redistribute => "Redistribute"; - public static string ReturnError => "ReturnError"; + public static string Return503Error => "Return503Error"; } } } diff --git a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityDefaultOptions.cs b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityDefaultOptions.cs index 39548cc04..0f680c245 100644 --- a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityDefaultOptions.cs +++ b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityDefaultOptions.cs @@ -11,7 +11,7 @@ namespace Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract public class SessionAffinityDefaultOptions { private string _defaultMode = SessionAffinityBuiltIns.Modes.Cookie; - private string _defaultMissingDestinationHandler = SessionAffinityBuiltIns.MissingDestinationHandlers.ReturnError; + private string _defaultAffinityFailurePolicy = SessionAffinityBuiltIns.AffinityFailurePolicies.Redistribute; /// /// Default session affinity mode to be used when none is specified for a backend. @@ -23,12 +23,12 @@ public string DefaultMode } /// - /// Default strategy handling missing destination for an affinitized request. + /// Default affinity failure handling policy. /// - public string MissingDestinationHandler + public string AffinityFailurePolicy { - get => _defaultMissingDestinationHandler; - set => _defaultMissingDestinationHandler = value ?? throw new ArgumentNullException(nameof(value)); + get => _defaultAffinityFailurePolicy; + set => _defaultAffinityFailurePolicy = value ?? throw new ArgumentNullException(nameof(value)); } /// diff --git a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityOptions.cs b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityOptions.cs index 22ef60256..300a0a099 100644 --- a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityOptions.cs +++ b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityOptions.cs @@ -19,7 +19,7 @@ public sealed class SessionAffinityOptions /// /// Strategy handling missing destination for an affinitized request. /// - public string MissingDestinationHandler { get; set; } + public string AffinityFailurePolicy { get; set; } /// /// Key-value pair collection holding extra settings specific to different affinity modes. @@ -31,7 +31,7 @@ internal SessionAffinityOptions DeepClone() return new SessionAffinityOptions { Mode = Mode, - MissingDestinationHandler = MissingDestinationHandler, + AffinityFailurePolicy = AffinityFailurePolicy, Settings = Settings?.DeepClone(StringComparer.Ordinal) }; } diff --git a/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs b/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs index 9607d7482..0ad855f72 100644 --- a/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs +++ b/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs @@ -88,16 +88,18 @@ public static IReverseProxyBuilder AddBackgroundWorkers(this IReverseProxyBuilde return builder; } - public static IReverseProxyBuilder AddSessionAffinityProvider(this IReverseProxyBuilder builder, bool enableForAllBackends) + public static IReverseProxyBuilder AddSessionAffinityProvider(this IReverseProxyBuilder builder) { - if (enableForAllBackends) - { - builder.Services.AddOptions().Configure(o => o.EnabledForAllBackends = true); - } + return builder.AddSessionAffinityProvider(_ => { }); + } + + public static IReverseProxyBuilder AddSessionAffinityProvider(this IReverseProxyBuilder builder, Action configureOptions) + { + builder.Services.AddOptions().Configure(configureOptions); builder.Services.TryAddEnumerable(new[] { - new ServiceDescriptor(typeof(IMissingDestinationHandler), typeof(PickRandomMissingDestinationHandler), ServiceLifetime.Singleton), - new ServiceDescriptor(typeof(IMissingDestinationHandler), typeof(ReturnErrorMissingDestinationHandler), ServiceLifetime.Singleton) + new ServiceDescriptor(typeof(IAffinityFailurePolicy), typeof(RedistributeAffinityFailurePolicy), ServiceLifetime.Singleton), + new ServiceDescriptor(typeof(IAffinityFailurePolicy), typeof(Return503ErrorAffinityFailurePolicy), ServiceLifetime.Singleton) }); builder.Services.TryAddEnumerable(new[] { new ServiceDescriptor(typeof(ISessionAffinityProvider), typeof(CookieSessionAffinityProvider), ServiceLifetime.Singleton), diff --git a/src/ReverseProxy/Configuration/DependencyInjection/ReverseProxyServiceCollectionExtensions.cs b/src/ReverseProxy/Configuration/DependencyInjection/ReverseProxyServiceCollectionExtensions.cs index 235b172d4..a4462253d 100644 --- a/src/ReverseProxy/Configuration/DependencyInjection/ReverseProxyServiceCollectionExtensions.cs +++ b/src/ReverseProxy/Configuration/DependencyInjection/ReverseProxyServiceCollectionExtensions.cs @@ -29,7 +29,7 @@ public static IReverseProxyBuilder AddReverseProxy(this IServiceCollection servi .AddRuntimeStateManagers() .AddConfigManager() .AddDynamicEndpointDataSource() - .AddSessionAffinityProvider(true) + .AddSessionAffinityProvider() .AddProxy() .AddBackgroundWorkers(); diff --git a/src/ReverseProxy/EventIds.cs b/src/ReverseProxy/EventIds.cs index b8dcd4208..7cf75a23a 100644 --- a/src/ReverseProxy/EventIds.cs +++ b/src/ReverseProxy/EventIds.cs @@ -40,7 +40,7 @@ internal static class EventIds public static readonly EventId OperationStarted = new EventId(31, "OperationStarted"); public static readonly EventId OperationEnded = new EventId(32, "OperationEnded"); public static readonly EventId OperationFailed = new EventId(33, "OperationFailed"); - public static readonly EventId AffinitizedDestinationIsNotFound = new EventId(34, "AffinitizedDestinationIsNotFound"); + public static readonly EventId AffinityResolutionFailedForBackend = new EventId(34, "AffinityResolutionFailedForBackend"); public static readonly EventId MultipleDestinationsOnBackendToEstablishRequestAffinity = new EventId(35, "MultipleDestinationsOnBackendToEstablishRequestAffinity"); public static readonly EventId AffinityCannotBeEstablishedBecauseNoDestinationsFoundOnBackend = new EventId(36, "AffinityCannotBeEstablishedBecauseNoDestinationsFoundOnBackend"); public static readonly EventId RequestAffinityToDestinationCannotBeEstablishedBecauseAffinitizationDisabled = new EventId(37, "RequestAffinityToDestinationCannotBeEstablishedBecauseAffinitizationDisabled"); diff --git a/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs b/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs index 0c34d2978..c1966d0be 100644 --- a/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs +++ b/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; +using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; using Microsoft.ReverseProxy.Abstractions.Telemetry; @@ -19,12 +20,14 @@ internal class AffinitizedDestinationLookupMiddleware { private readonly RequestDelegate _next; private readonly IDictionary _sessionAffinityProviders; + private readonly IDictionary _affinityFailurePolicies; private readonly IOperationLogger _operationLogger; private readonly ILogger _logger; public AffinitizedDestinationLookupMiddleware( RequestDelegate next, IEnumerable sessionAffinityProviders, + IEnumerable affinityFailurePolicies, IOperationLogger operationLogger, ILogger logger) { @@ -32,9 +35,10 @@ public AffinitizedDestinationLookupMiddleware( _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _operationLogger = operationLogger ?? throw new ArgumentNullException(nameof(logger)); _sessionAffinityProviders = sessionAffinityProviders.ToProviderDictionary(); + _affinityFailurePolicies = affinityFailurePolicies?.ToPolicyDictionary() ?? throw new ArgumentNullException(nameof(affinityFailurePolicies)); } - public Task Invoke(HttpContext context) + public async Task Invoke(HttpContext context) { var backend = context.GetRequiredBackend(); var destinationsFeature = context.GetRequiredDestinationFeature(); @@ -44,45 +48,54 @@ public Task Invoke(HttpContext context) if (options.Enabled) { - var affinitizedDestinations = _operationLogger.Execute( + var affinityResult = _operationLogger.Execute( "ReverseProxy.FindAffinitizedDestinations", - () => FindAffinitizedDestinations(context, destinations, backend, options)); - if (affinitizedDestinations.DestinationsFound) + () => { + var currentProvider = _sessionAffinityProviders.GetRequiredServiceById(options.Mode); + return currentProvider.FindAffinitizedDestinations(context, destinations, backend.BackendId, options); + }); + switch (affinityResult.Status) { - if (affinitizedDestinations.Result.Destinations.Count > 0) - { - destinations = affinitizedDestinations.Result.Destinations; - destinationsFeature.Destinations = destinations; - } - else - { - Log.AffinitizedDestinationIsNotFound(_logger, backend.BackendId); - context.Response.StatusCode = 503; - return Task.CompletedTask; - } + case AffinityStatus.OK: + destinationsFeature.Destinations = affinityResult.Destinations; + break; + // This implementation treat them the same. + case AffinityStatus.AffinityDisabled: + case AffinityStatus.AffinityKeyNotSet: + // Nothing to do so just continue processing + break; + case AffinityStatus.AffinityKeyExtractionFailed: + case AffinityStatus.DestinationNotFound: + await _operationLogger.ExecuteAsync("ReverseProxy.HandleAffinityFailure", async () => + { + var failurePolicy = _affinityFailurePolicies.GetRequiredServiceById(options.AffinityFailurePolicy); + if (!await failurePolicy.Handle(context, options, affinityResult.Status)) + { + // Policy reported the failure is unrecoverable and took the full responsibility for its handling, + // so we simply stop processing. + Log.AffinityResolutionFailedForBackend(_logger, backend.BackendId); + return; + } + }); + break; + default: + throw new NotSupportedException($"Affinity status '{affinityResult.Status}' is not supported."); } } - return _next(context); - } - - private (bool DestinationsFound, AffinityResult Result) FindAffinitizedDestinations(HttpContext context, IReadOnlyList destinations, BackendInfo backend, BackendConfig.BackendSessionAffinityOptions options) - { - var currentProvider = _sessionAffinityProviders.GetRequiredServiceById(options.Mode); - var destinationsFound = currentProvider.TryFindAffinitizedDestinations(context, destinations, backend.BackendId, options, out var affinityResult); - return (destinationsFound, affinityResult); + await _next(context); } private static class Log { - private static readonly Action _affinitizedDestinationIsNotFound = LoggerMessage.Define( + private static readonly Action _affinityResolutioFailedForBackend = LoggerMessage.Define( LogLevel.Warning, - EventIds.AffinitizedDestinationIsNotFound, - "No destinations found for the affinitized request on backend `{backendId}`."); + EventIds.AffinityResolutionFailedForBackend, + "Affinity resolution failed for backend `{backendId}`."); - public static void AffinitizedDestinationIsNotFound(ILogger logger, string backendId) + public static void AffinityResolutionFailedForBackend(ILogger logger, string backendId) { - _affinitizedDestinationIsNotFound(logger, backendId, null); + _affinityResolutioFailedForBackend(logger, backendId, null); } } } diff --git a/src/ReverseProxy/Service/Config/ConfigErrors.cs b/src/ReverseProxy/Service/Config/ConfigErrors.cs index 304e25d74..1258e2973 100644 --- a/src/ReverseProxy/Service/Config/ConfigErrors.cs +++ b/src/ReverseProxy/Service/Config/ConfigErrors.cs @@ -24,7 +24,7 @@ internal static class ConfigErrors internal const string ConfigBuilderBackendIdMismatch = "ConfigBuilder_BackendIdMismatch"; internal const string ConfigBuilderBackendNoProviderFoundForSessionAffinityMode = "ConfigBuilder_BackendNoProviderFoundForSessionAffinityMode"; - internal const string ConfigBuilderBackendNoMissingDestinationHandlerFoundForSpecifiedName = "ConfigBuilder_NoMissingDestinationHandlerFoundForSpecifiedName"; + internal const string ConfigBuilderBackendNoAffinityFailurePolicyFoundForSpecifiedName = "ConfigBuilder_NoAffinityFailurePolicyFoundForSpecifiedName"; internal const string ConfigBuilderBackendException = "ConfigBuilder_BackendException"; internal const string ConfigBuilderRouteException = "ConfigBuilder_RouteException"; } diff --git a/src/ReverseProxy/Service/Config/DynamicConfigBuilder.cs b/src/ReverseProxy/Service/Config/DynamicConfigBuilder.cs index 44c851cfe..b5d861603 100644 --- a/src/ReverseProxy/Service/Config/DynamicConfigBuilder.cs +++ b/src/ReverseProxy/Service/Config/DynamicConfigBuilder.cs @@ -21,7 +21,7 @@ internal class DynamicConfigBuilder : IDynamicConfigBuilder private readonly IRoutesRepo _routesRepo; private readonly IRouteValidator _parsedRouteValidator; private readonly IDictionary _sessionAffinityProviders; - private readonly IDictionary _missingDestionationHandlers; + private readonly IDictionary _affinityFailurePolicies; private readonly SessionAffinityDefaultOptions _sessionAffinityDefaultOptions; public DynamicConfigBuilder( @@ -30,7 +30,7 @@ public DynamicConfigBuilder( IRoutesRepo routesRepo, IRouteValidator parsedRouteValidator, IEnumerable sessionAffinityProviders, - IEnumerable missingDestinationHandlers, + IEnumerable affinityFailurePolicies, IOptions sessionAffinityDefaultOptions) { Contracts.CheckValue(filters, nameof(filters)); @@ -38,13 +38,13 @@ public DynamicConfigBuilder( Contracts.CheckValue(routesRepo, nameof(routesRepo)); Contracts.CheckValue(parsedRouteValidator, nameof(parsedRouteValidator)); Contracts.CheckValue(sessionAffinityProviders, nameof(sessionAffinityProviders)); - Contracts.CheckValue(missingDestinationHandlers, nameof(missingDestinationHandlers)); + Contracts.CheckValue(affinityFailurePolicies, nameof(affinityFailurePolicies)); _filters = filters; _backendsRepo = backendsRepo; _routesRepo = routesRepo; _parsedRouteValidator = parsedRouteValidator; _sessionAffinityProviders = sessionAffinityProviders.ToProviderDictionary(); - _missingDestionationHandlers = missingDestinationHandlers.ToHandlerDictionary(); + _affinityFailurePolicies = affinityFailurePolicies.ToPolicyDictionary(); _sessionAffinityDefaultOptions = sessionAffinityDefaultOptions?.Value ?? throw new ArgumentNullException(nameof(sessionAffinityDefaultOptions)); } @@ -124,15 +124,15 @@ private void ValidateSessionAffinity(IConfigErrorReporter errorReporter, string errorReporter.ReportError(ConfigErrors.ConfigBuilderBackendNoProviderFoundForSessionAffinityMode, id, $"No matching {nameof(ISessionAffinityProvider)} found for the session affinity mode {affinityMode} set on the backend {backend.Id}."); } - if (string.IsNullOrEmpty(backend.SessionAffinity.MissingDestinationHandler)) + if (string.IsNullOrEmpty(backend.SessionAffinity.AffinityFailurePolicy)) { - backend.SessionAffinity.MissingDestinationHandler = _sessionAffinityDefaultOptions.MissingDestinationHandler; + backend.SessionAffinity.AffinityFailurePolicy = _sessionAffinityDefaultOptions.AffinityFailurePolicy; } - var missingDestinationHandler = backend.SessionAffinity.MissingDestinationHandler; - if (!_missingDestionationHandlers.ContainsKey(missingDestinationHandler)) + var affinityFailurePolicy = backend.SessionAffinity.AffinityFailurePolicy; + if (!_affinityFailurePolicies.ContainsKey(affinityFailurePolicy)) { - errorReporter.ReportError(ConfigErrors.ConfigBuilderBackendNoMissingDestinationHandlerFoundForSpecifiedName, id, $"No matching {nameof(IMissingDestinationHandler)} found for the missing affinitizated destination handler name {missingDestinationHandler} set on the backend {backend.Id}."); + errorReporter.ReportError(ConfigErrors.ConfigBuilderBackendNoAffinityFailurePolicyFoundForSpecifiedName, id, $"No matching {nameof(IAffinityFailurePolicy)} found for the affinity failure policy name {affinityFailurePolicy} set on the backend {backend.Id}."); } } diff --git a/src/ReverseProxy/Service/Management/ReverseProxyConfigManager.cs b/src/ReverseProxy/Service/Management/ReverseProxyConfigManager.cs index b3a22c6ff..925cfe147 100644 --- a/src/ReverseProxy/Service/Management/ReverseProxyConfigManager.cs +++ b/src/ReverseProxy/Service/Management/ReverseProxyConfigManager.cs @@ -98,7 +98,7 @@ private void UpdateRuntimeBackends(DynamicConfigRoot config) new BackendConfig.BackendSessionAffinityOptions( enabled: configBackend.SessionAffinity != null, mode: configBackend.SessionAffinity?.Mode, - missingDestinationHandler: configBackend.SessionAffinity?.MissingDestinationHandler, + affinityFailurePolicy: configBackend.SessionAffinity?.AffinityFailurePolicy, settings: configBackend.SessionAffinity?.Settings as IReadOnlyDictionary)); var currentBackendConfig = backend.Config.Value; diff --git a/src/ReverseProxy/Service/RuntimeModel/BackendConfig.cs b/src/ReverseProxy/Service/RuntimeModel/BackendConfig.cs index 0b4d47f6c..b7e87fb72 100644 --- a/src/ReverseProxy/Service/RuntimeModel/BackendConfig.cs +++ b/src/ReverseProxy/Service/RuntimeModel/BackendConfig.cs @@ -96,10 +96,10 @@ public BackendLoadBalancingOptions(LoadBalancingMode mode) public readonly struct BackendSessionAffinityOptions { - public BackendSessionAffinityOptions(bool enabled, string mode, string missingDestinationHandler, IReadOnlyDictionary settings) + public BackendSessionAffinityOptions(bool enabled, string mode, string affinityFailurePolicy, IReadOnlyDictionary settings) { Mode = mode; - MissingDestinationHandler = missingDestinationHandler; + AffinityFailurePolicy = affinityFailurePolicy; Settings = settings; Enabled = enabled; } @@ -108,7 +108,7 @@ public BackendSessionAffinityOptions(bool enabled, string mode, string missingDe public string Mode { get; } - public string MissingDestinationHandler { get; } + public string AffinityFailurePolicy { get; } public IReadOnlyDictionary Settings { get; } } diff --git a/src/ReverseProxy/Service/SessionAffinity/AffinityResult.cs b/src/ReverseProxy/Service/SessionAffinity/AffinityResult.cs index 42e0c2d1f..360d07a5b 100644 --- a/src/ReverseProxy/Service/SessionAffinity/AffinityResult.cs +++ b/src/ReverseProxy/Service/SessionAffinity/AffinityResult.cs @@ -6,13 +6,19 @@ namespace Microsoft.ReverseProxy.Service.SessionAffinity { + /// + /// Affinity resolution result. + /// public readonly struct AffinityResult { public IReadOnlyList Destinations { get; } - public AffinityResult(IReadOnlyList destinations) + public AffinityStatus Status { get; } + + public AffinityResult(IReadOnlyList destinations, AffinityStatus status) { Destinations = destinations; + Status = status; } } } diff --git a/src/ReverseProxy/Service/SessionAffinity/AffinityStatus.cs b/src/ReverseProxy/Service/SessionAffinity/AffinityStatus.cs new file mode 100644 index 000000000..d60e8d473 --- /dev/null +++ b/src/ReverseProxy/Service/SessionAffinity/AffinityStatus.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.ReverseProxy.Service.SessionAffinity +{ + /// + /// Affinity resolution status. + /// + public enum AffinityStatus + { + OK, + AffinityDisabled, + AffinityKeyNotSet, + AffinityKeyExtractionFailed, + DestinationNotFound + } +} diff --git a/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs index 2e14e2aae..345f7e1c3 100644 --- a/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Text; using Microsoft.AspNetCore.DataProtection; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; @@ -12,15 +13,13 @@ namespace Microsoft.ReverseProxy.Service.SessionAffinity { internal abstract class BaseSessionAffinityProvider : ISessionAffinityProvider { + private readonly IDataProtector _dataProtector; protected static readonly object AffinityKeyId = new object(); - protected readonly IDataProtector DataProtector; - protected readonly IDictionary MissingDestinationHandlers; protected readonly ILogger Logger; - protected BaseSessionAffinityProvider(IDataProtectionProvider dataProtectionProvider, IEnumerable missingDestinationHandlers, ILogger logger) + protected BaseSessionAffinityProvider(IDataProtectionProvider dataProtectionProvider, ILogger logger) { - DataProtector = dataProtectionProvider?.CreateProtector(GetType().FullName) ?? throw new ArgumentNullException(nameof(dataProtectionProvider)); - MissingDestinationHandlers = missingDestinationHandlers?.ToHandlerDictionary() ?? throw new ArgumentNullException(nameof(missingDestinationHandlers)); + _dataProtector = dataProtectionProvider?.CreateProtector(GetType().FullName) ?? throw new ArgumentNullException(nameof(dataProtectionProvider)); Logger = logger ?? throw new ArgumentNullException(nameof(logger)); } @@ -34,7 +33,8 @@ public virtual void AffinitizeRequest(HttpContext context, in BackendConfig.Back return; } - if (!context.Items.TryGetValue(AffinityKeyId, out var affinityKey)) // If affinity key is already set on request, we assume that passed destination always matches to that key + // If affinity key is already set on request, we assume that passed destination always matches to that key + if (!context.Items.TryGetValue(AffinityKeyId, out var affinityKey)) { affinityKey = GetDestinationAffinityKey(destination); } @@ -42,20 +42,19 @@ public virtual void AffinitizeRequest(HttpContext context, in BackendConfig.Back SetAffinityKey(context, options, (T)affinityKey); } - public virtual bool TryFindAffinitizedDestinations(HttpContext context, IReadOnlyList destinations, string backendId, in BackendConfig.BackendSessionAffinityOptions options, out AffinityResult affinityResult) + public virtual AffinityResult FindAffinitizedDestinations(HttpContext context, IReadOnlyList destinations, string backendId, in BackendConfig.BackendSessionAffinityOptions options) { if (!options.Enabled) { - affinityResult = default; - return false; + // This case is handled separately to improve the type autonomy and the pipeline extensibility + return new AffinityResult(null, AffinityStatus.AffinityDisabled); } var requestAffinityKey = GetRequestAffinityKey(context, options); - if (requestAffinityKey == null) + if (requestAffinityKey.Key == null) { - affinityResult = default; - return false; + return new AffinityResult(null, requestAffinityKey.ExtractedSuccessfully ? AffinityStatus.AffinityKeyNotSet : AffinityStatus.AffinityKeyExtractionFailed); } var matchingDestinations = new DestinationInfo[1]; @@ -82,16 +81,10 @@ public virtual bool TryFindAffinitizedDestinations(HttpContext context, IReadOnl // Empty destination list passed to this method is handled the same way as if no matching destinations are found. if (matchingDestinations[0] == null) { - var failureHandler = MissingDestinationHandlers[options.MissingDestinationHandler]; - var newAffinitizedDestinations = failureHandler.Handle(context, options, requestAffinityKey, destinations); - affinityResult = new AffinityResult(newAffinitizedDestinations); - } - else - { - affinityResult = new AffinityResult(matchingDestinations); + return new AffinityResult(null, AffinityStatus.DestinationNotFound); } - return true; + return new AffinityResult(matchingDestinations, AffinityStatus.OK); } protected virtual string GetSettingValue(string key, BackendConfig.BackendSessionAffinityOptions options) @@ -106,10 +99,65 @@ protected virtual string GetSettingValue(string key, BackendConfig.BackendSessio protected abstract T GetDestinationAffinityKey(DestinationInfo destination); - protected abstract T GetRequestAffinityKey(HttpContext context, in BackendConfig.BackendSessionAffinityOptions options); + protected abstract (T Key, bool ExtractedSuccessfully) GetRequestAffinityKey(HttpContext context, in BackendConfig.BackendSessionAffinityOptions options); protected abstract void SetAffinityKey(HttpContext context, in BackendConfig.BackendSessionAffinityOptions options, T unencryptedKey); + protected string Protect(string unencryptedKey) + { + if (string.IsNullOrEmpty(unencryptedKey)) + { + return unencryptedKey; + } + + var userData = Encoding.UTF8.GetBytes(unencryptedKey); + + var protectedData = _dataProtector.Protect(userData); + return Convert.ToBase64String(protectedData).TrimEnd('='); + } + + protected (string Key, bool ExtractedSuccessfully) Unprotect(string encryptedRequestKey) + { + if (string.IsNullOrEmpty(encryptedRequestKey)) + { + return (Key: null, ExtractedSuccessfully: true); + } + + try + { + var keyBytes = Convert.FromBase64String(Pad(encryptedRequestKey)); + if (keyBytes == null) + { + Log.RequestAffinityKeyCookieCannotBeDecodedFromBase64(Logger); + return (Key: null, ExtractedSuccessfully: false); + } + + var decryptedKeyBytes = _dataProtector.Unprotect(keyBytes); + if (decryptedKeyBytes == null) + { + Log.RequestAffinityKeyCookieDecryptionFailed(Logger, null); + return (Key: null, ExtractedSuccessfully: false); + } + + return (Key: Encoding.UTF8.GetString(decryptedKeyBytes), ExtractedSuccessfully: true); + } + catch (Exception ex) + { + Log.RequestAffinityKeyCookieDecryptionFailed(Logger, ex); + return (Key: null, ExtractedSuccessfully: false); + } + } + + private static string Pad(string text) + { + var padding = 3 - ((text.Length + 3) % 4); + if (padding == 0) + { + return text; + } + return text + new string('=', padding); + } + private static class Log { private static readonly Action _affinityCannotBeEstablishedBecauseNoDestinationsFound = LoggerMessage.Define( @@ -122,6 +170,16 @@ private static class Log EventIds.RequestAffinityToDestinationCannotBeEstablishedBecauseAffinitizationDisabled, "The request affinity to destination `{destinationId}` cannot be established because affinitization is disabled for the backend."); + private static readonly Action _requestAffinityKeyCookieDecryptionFailed = LoggerMessage.Define( + LogLevel.Error, + EventIds.RequestAffinityKeyCookieDecryptionFailed, + "The request affinity key cookie decryption failed."); + + private static readonly Action _requestAffinityKeyCookieCannotBeDecodedFromBase64 = LoggerMessage.Define( + LogLevel.Error, + EventIds.RequestAffinityKeyCookieCannotBeDecodedFromBase64, + "The request affinity key cookie cannot be decoded from Base64 representation."); + public static void AffinityCannotBeEstablishedBecauseNoDestinationsFound(ILogger logger, string backendId) { _affinityCannotBeEstablishedBecauseNoDestinationsFound(logger, backendId, null); @@ -131,6 +189,16 @@ public static void RequestAffinityToDestinationCannotBeEstablishedBecauseAffinit { _requestAffinityToDestinationCannotBeEstablishedBecauseAffinitizationDisabled(logger, destinationId, null); } + + public static void RequestAffinityKeyCookieDecryptionFailed(ILogger logger, Exception ex) + { + _requestAffinityKeyCookieDecryptionFailed(logger, ex); + } + + public static void RequestAffinityKeyCookieCannotBeDecodedFromBase64(ILogger logger) + { + _requestAffinityKeyCookieCannotBeDecodedFromBase64(logger, null); + } } } } diff --git a/src/ReverseProxy/Service/SessionAffinity/CookieSessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/CookieSessionAffinityProvider.cs index 965f58837..0fa930b39 100644 --- a/src/ReverseProxy/Service/SessionAffinity/CookieSessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/CookieSessionAffinityProvider.cs @@ -7,9 +7,7 @@ using Microsoft.Extensions.Options; using Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract; using Microsoft.ReverseProxy.RuntimeModel; -using System.Collections.Generic; using Microsoft.Extensions.Logging; -using System.Text; namespace Microsoft.ReverseProxy.Service.SessionAffinity { @@ -20,9 +18,8 @@ internal class CookieSessionAffinityProvider : BaseSessionAffinityProvider providerOptions, IDataProtectionProvider dataProtectionProvider, - IEnumerable missingDestinationHandlers, ILogger logger) - : base(dataProtectionProvider, missingDestinationHandlers, logger) + : base(dataProtectionProvider, logger) { _providerOptions = providerOptions?.Value ?? throw new ArgumentNullException(nameof(providerOptions)); } @@ -34,10 +31,10 @@ protected override string GetDestinationAffinityKey(DestinationInfo destination) return destination.DestinationId; } - protected override string GetRequestAffinityKey(HttpContext context, in BackendConfig.BackendSessionAffinityOptions options) + protected override (string Key, bool ExtractedSuccessfully) GetRequestAffinityKey(HttpContext context, in BackendConfig.BackendSessionAffinityOptions options) { var encryptedRequestKey = context.Request.Cookies.TryGetValue(_providerOptions.Cookie.Name, out var keyInCookie) ? keyInCookie : null; - return !string.IsNullOrEmpty(encryptedRequestKey) ? Unprotect(encryptedRequestKey) : null; + return Unprotect(encryptedRequestKey); } protected override void SetAffinityKey(HttpContext context, in BackendConfig.BackendSessionAffinityOptions options, string unencryptedKey) @@ -45,78 +42,5 @@ protected override void SetAffinityKey(HttpContext context, in BackendConfig.Bac var affinityCookieOptions = _providerOptions.Cookie.Build(context); context.Response.Cookies.Append(_providerOptions.Cookie.Name, Protect(unencryptedKey), affinityCookieOptions); } - - private string Protect(string unencryptedKey) - { - if (string.IsNullOrEmpty(unencryptedKey)) - { - return unencryptedKey; - } - - var userData = Encoding.UTF8.GetBytes(unencryptedKey); - - var protectedData = DataProtector.Protect(userData); - return Convert.ToBase64String(protectedData).TrimEnd('='); - } - - private string Unprotect(string encryptedRequestKey) - { - try - { - var keyBytes = Convert.FromBase64String(Pad(encryptedRequestKey)); - if (keyBytes == null) - { - Log.RequestAffinityKeyCookieCannotBeDecodedFromBase64(Logger); - return null; - } - - var decryptedKeyBytes = DataProtector.Unprotect(keyBytes); - if (decryptedKeyBytes == null) - { - Log.RequestAffinityKeyCookieDecryptionFailed(Logger, null); - return null; - } - - return Encoding.UTF8.GetString(decryptedKeyBytes); - } - catch (Exception ex) - { - Log.RequestAffinityKeyCookieDecryptionFailed(Logger, ex); - return null; - } - } - - private static string Pad(string text) - { - var padding = 3 - ((text.Length + 3) % 4); - if (padding == 0) - { - return text; - } - return text + new string('=', padding); - } - - private static class Log - { - private static readonly Action _requestAffinityKeyCookieDecryptionFailed = LoggerMessage.Define( - LogLevel.Error, - EventIds.RequestAffinityKeyCookieDecryptionFailed, - "The request affinity key cookie decryption failed."); - - private static readonly Action _requestAffinityKeyCookieCannotBeDecodedFromBase64 = LoggerMessage.Define( - LogLevel.Error, - EventIds.RequestAffinityKeyCookieCannotBeDecodedFromBase64, - "The request affinity key cookie cannot be decoded from Base64 representation."); - - public static void RequestAffinityKeyCookieDecryptionFailed(ILogger logger, Exception ex) - { - _requestAffinityKeyCookieDecryptionFailed(logger, ex); - } - - public static void RequestAffinityKeyCookieCannotBeDecodedFromBase64(ILogger logger) - { - _requestAffinityKeyCookieCannotBeDecodedFromBase64(logger, null); - } - } } } diff --git a/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs index 9119b49d6..242b95bd1 100644 --- a/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -using System.Collections.Generic; using Microsoft.AspNetCore.DataProtection; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; @@ -17,9 +16,8 @@ internal class CustomHeaderSessionAffinityProvider : BaseSessionAffinityProvider public CustomHeaderSessionAffinityProvider( IDataProtectionProvider dataProtectionProvider, - IEnumerable missingDestinationHandlers, ILogger logger) - : base(dataProtectionProvider, missingDestinationHandlers, logger) + : base(dataProtectionProvider, logger) {} public override string Mode => SessionAffinityBuiltIns.Modes.CustomHeander; @@ -29,18 +27,18 @@ protected override string GetDestinationAffinityKey(DestinationInfo destination) return destination.DestinationId; } - protected override string GetRequestAffinityKey(HttpContext context, in BackendConfig.BackendSessionAffinityOptions options) + protected override (string Key, bool ExtractedSuccessfully) GetRequestAffinityKey(HttpContext context, in BackendConfig.BackendSessionAffinityOptions options) { var customHeaderName = GetSettingValue(CustomHeaderNameKey, options); var keyHeaderValues = context.Request.Headers[customHeaderName]; var encryptedRequestKey = !StringValues.IsNullOrEmpty(keyHeaderValues) ? keyHeaderValues[0] : null; // We always take the first value of a custom header storing an affinity key - return encryptedRequestKey != null ? DataProtector.Unprotect(encryptedRequestKey) : null; + return Unprotect(encryptedRequestKey); } protected override void SetAffinityKey(HttpContext context, in BackendConfig.BackendSessionAffinityOptions options, string unencryptedKey) { var customHeaderName = GetSettingValue(CustomHeaderNameKey, options); - context.Response.Headers.Append(customHeaderName, DataProtector.Protect(unencryptedKey)); + context.Response.Headers.Append(customHeaderName, Protect(unencryptedKey)); } } } diff --git a/src/ReverseProxy/Service/SessionAffinity/IAffinityFailurePolicy.cs b/src/ReverseProxy/Service/SessionAffinity/IAffinityFailurePolicy.cs new file mode 100644 index 000000000..97f8a9dbe --- /dev/null +++ b/src/ReverseProxy/Service/SessionAffinity/IAffinityFailurePolicy.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.ReverseProxy.Middleware; +using Microsoft.ReverseProxy.RuntimeModel; + +namespace Microsoft.ReverseProxy.Service.SessionAffinity +{ + /// + /// Affinity failures handling policy. + /// + internal interface IAffinityFailurePolicy + { + /// + /// A unique identifier for this failure policy. This will be referenced from config. + /// + public string Name { get; } + + /// + /// Handles affinity failures. This method assumes the full control on + /// and can change it in any way. + /// + /// Current request's context. + /// Session affinity options set for the backend. + /// Affinity resolution status. + /// + /// if the failure has been considered recoverable and the request processing can proceed. + /// Otherwise, indicating that the request's processing must be terminated. + /// + public Task Handle(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, AffinityStatus affinityStatus); + } +} diff --git a/src/ReverseProxy/Service/SessionAffinity/IMissingDestinationHandler.cs b/src/ReverseProxy/Service/SessionAffinity/IMissingDestinationHandler.cs deleted file mode 100644 index 4e19f357a..000000000 --- a/src/ReverseProxy/Service/SessionAffinity/IMissingDestinationHandler.cs +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System.Collections.Generic; -using Microsoft.AspNetCore.Http; -using Microsoft.ReverseProxy.RuntimeModel; - -namespace Microsoft.ReverseProxy.Service.SessionAffinity -{ - /// - /// Handles failures caused by a missing for an affinizied request. - /// - internal interface IMissingDestinationHandler - { - /// - /// A unique identifier for this missing destionation handler implementation. This will be referenced from config. - /// - public string Name { get; } - - /// - /// Handles destination affinitization failure when no was found for the given request's affinity key. - /// - /// Current request's context. - /// Request's affinity key. - /// s available for the request. - /// List of chosen to be affinitized to the request. - public IReadOnlyList Handle(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, object affinityKey, IReadOnlyList availableDestinations); - } -} diff --git a/src/ReverseProxy/Service/SessionAffinity/ISessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/ISessionAffinityProvider.cs index 6dfdd690e..a5799a20e 100644 --- a/src/ReverseProxy/Service/SessionAffinity/ISessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/ISessionAffinityProvider.cs @@ -19,15 +19,14 @@ public interface ISessionAffinityProvider public string Mode { get; } /// - /// Tries to find to which the current request is affinitized by the affinity key. + /// Finds to which the current request is affinitized by the affinity key. /// /// Current request's context. /// s available for the request. /// Target backend ID. /// Affinity options. - /// Affinitized s found for the request. - /// if affinitized s were successfully found, otherwise . - public bool TryFindAffinitizedDestinations(HttpContext context, IReadOnlyList destinations, string backendId, in BackendConfig.BackendSessionAffinityOptions options, out AffinityResult affinityResult); + /// carrying the found affinitized destinations if any and the . + public AffinityResult FindAffinitizedDestinations(HttpContext context, IReadOnlyList destinations, string backendId, in BackendConfig.BackendSessionAffinityOptions options); /// /// Affinitize the current request to the given by setting the affinity key extracted from . diff --git a/src/ReverseProxy/Service/SessionAffinity/PickRandomMissingDestinationHandler.cs b/src/ReverseProxy/Service/SessionAffinity/PickRandomMissingDestinationHandler.cs index 831eece91..c9946fa56 100644 --- a/src/ReverseProxy/Service/SessionAffinity/PickRandomMissingDestinationHandler.cs +++ b/src/ReverseProxy/Service/SessionAffinity/PickRandomMissingDestinationHandler.cs @@ -1,29 +1,22 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -using System; -using System.Collections.Generic; +using System.Threading.Tasks; using Microsoft.AspNetCore.Http; using Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract; using Microsoft.ReverseProxy.RuntimeModel; namespace Microsoft.ReverseProxy.Service.SessionAffinity { - internal class PickRandomMissingDestinationHandler : IMissingDestinationHandler + internal class RedistributeAffinityFailurePolicy : IAffinityFailurePolicy { - private readonly Random _random = new Random(); + public string Name => SessionAffinityBuiltIns.AffinityFailurePolicies.Redistribute; - public string Name => SessionAffinityBuiltIns.MissingDestinationHandlers.PickRandom; - - public IReadOnlyList Handle(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, object affinityKey, IReadOnlyList availableDestinations) + public Task Handle(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, AffinityStatus affinityStatus) { - if (availableDestinations.Count == 0) - { - return availableDestinations; - } - - var index = _random.Next(availableDestinations.Count); - return new[] { availableDestinations[index] }; + // Available destinations list have not been changed in the context, + // so simply allow processing to proceed to load balancing. + return Task.FromResult(true); } } } diff --git a/src/ReverseProxy/Service/SessionAffinity/ReturnErrorMissingDestinationHandler.cs b/src/ReverseProxy/Service/SessionAffinity/ReturnErrorMissingDestinationHandler.cs index 2caf63e0c..e7b1fa2af 100644 --- a/src/ReverseProxy/Service/SessionAffinity/ReturnErrorMissingDestinationHandler.cs +++ b/src/ReverseProxy/Service/SessionAffinity/ReturnErrorMissingDestinationHandler.cs @@ -1,20 +1,21 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -using System.Collections.Generic; +using System.Threading.Tasks; using Microsoft.AspNetCore.Http; using Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract; using Microsoft.ReverseProxy.RuntimeModel; namespace Microsoft.ReverseProxy.Service.SessionAffinity { - internal class ReturnErrorMissingDestinationHandler : IMissingDestinationHandler + internal class Return503ErrorAffinityFailurePolicy : IAffinityFailurePolicy { - public string Name => SessionAffinityBuiltIns.MissingDestinationHandlers.ReturnError; + public string Name => SessionAffinityBuiltIns.AffinityFailurePolicies.Return503Error; - public IReadOnlyList Handle(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, object affinityKey, IReadOnlyList availableDestinations) + public Task Handle(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, AffinityStatus affinityStatus) { - return new DestinationInfo[0]; + context.Response.StatusCode = 503; + return Task.FromResult(true); } } } diff --git a/src/ReverseProxy/Service/SessionAffinity/SessionAffinityMiddlewareHelper.cs b/src/ReverseProxy/Service/SessionAffinity/SessionAffinityMiddlewareHelper.cs index c918e37df..bd73c6384 100644 --- a/src/ReverseProxy/Service/SessionAffinity/SessionAffinityMiddlewareHelper.cs +++ b/src/ReverseProxy/Service/SessionAffinity/SessionAffinityMiddlewareHelper.cs @@ -34,9 +34,9 @@ public static IDictionary ToProviderDictionary return ToDictionaryById(sessionAffinityProviders, p => p.Mode); } - public static IDictionary ToHandlerDictionary(this IEnumerable missingDestinationHandlers) + public static IDictionary ToPolicyDictionary(this IEnumerable affinityFailurePolicies) { - return ToDictionaryById(missingDestinationHandlers, p => p.Name); + return ToDictionaryById(affinityFailurePolicies, p => p.Name); } public static T GetRequiredServiceById(this IDictionary services, string id) diff --git a/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTest.cs b/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTest.cs index 5e51a9f48..3bf7fd4b3 100644 --- a/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTest.cs +++ b/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTest.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -using System.Collections.Generic; using Microsoft.AspNetCore.DataProtection; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; @@ -16,8 +15,8 @@ private class SessionAffinityProviderStub : BaseSessionAffinityProvider { public static readonly string AffinityKeyItemName = "StubAffinityKey"; - public SessionAffinityProviderStub(IDataProtectionProvider dataProtectionProvider, IEnumerable missingDestinationHandlers, ILogger logger) - : base(dataProtectionProvider, missingDestinationHandlers, logger) + public SessionAffinityProviderStub(IDataProtectionProvider dataProtectionProvider, ILogger logger) + : base(dataProtectionProvider, logger) {} public override string Mode => "Stub"; @@ -27,9 +26,9 @@ protected override string GetDestinationAffinityKey(DestinationInfo destination) return destination.DestinationId; } - protected override string GetRequestAffinityKey(HttpContext context, in BackendConfig.BackendSessionAffinityOptions options) + protected override (string Key, bool ExtractedSuccessfully) GetRequestAffinityKey(HttpContext context, in BackendConfig.BackendSessionAffinityOptions options) { - return (string)context.Items[AffinityKeyItemName]; + return (Key: (string)context.Items[AffinityKeyItemName], ExtractedSuccessfully: true); } protected override void SetAffinityKey(HttpContext context, in BackendConfig.BackendSessionAffinityOptions options, string unencryptedKey) From b63d8ac7bdf085d8291cdf525d961d9f8a53ddb6 Mon Sep 17 00:00:00 2001 From: Alexander Nikolaev Date: Thu, 28 May 2020 18:46:29 +0200 Subject: [PATCH 12/30] Affinity key is set on response only if it is a new affinity --- .../Middleware/AffinitizeRequestMiddleware.cs | 3 ++- .../SessionAffinity/BaseSessionAffinityProvider.cs | 12 +++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs b/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs index 7263f933e..9b0b6645c 100644 --- a/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs +++ b/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs @@ -63,7 +63,8 @@ private IReadOnlyList AffinitizeRequest(HttpContext context, Ba if (result.Count > 1) { Log.MultipleDestinationsOnBackendToEstablishRequestAffinity(_logger, backend.BackendId); - var singleDestination = destinations[_random.Next(destinations.Count)]; // It's assumed that all of them match to the request's affinity key. + // It's assumed that all of them match to the request's affinity key. + var singleDestination = destinations[_random.Next(destinations.Count)]; result = new[] { singleDestination }; } diff --git a/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs index 345f7e1c3..8989d990a 100644 --- a/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs @@ -33,13 +33,12 @@ public virtual void AffinitizeRequest(HttpContext context, in BackendConfig.Back return; } - // If affinity key is already set on request, we assume that passed destination always matches to that key - if (!context.Items.TryGetValue(AffinityKeyId, out var affinityKey)) + // Affinity key is set on the response only if it's a new affinity. + if (!context.Items.ContainsKey(AffinityKeyId)) { - affinityKey = GetDestinationAffinityKey(destination); + var affinityKey = GetDestinationAffinityKey(destination); + SetAffinityKey(context, options, affinityKey); } - - SetAffinityKey(context, options, (T)affinityKey); } public virtual AffinityResult FindAffinitizedDestinations(HttpContext context, IReadOnlyList destinations, string backendId, in BackendConfig.BackendSessionAffinityOptions options) @@ -60,8 +59,6 @@ public virtual AffinityResult FindAffinitizedDestinations(HttpContext context, I var matchingDestinations = new DestinationInfo[1]; if (destinations.Count > 0) { - context.Items[AffinityKeyId] = requestAffinityKey; - for (var i = 0; i < destinations.Count; i++) { if (requestAffinityKey.Equals(GetDestinationAffinityKey(destinations[i]))) @@ -84,6 +81,7 @@ public virtual AffinityResult FindAffinitizedDestinations(HttpContext context, I return new AffinityResult(null, AffinityStatus.DestinationNotFound); } + context.Items[AffinityKeyId] = requestAffinityKey; return new AffinityResult(matchingDestinations, AffinityStatus.OK); } From 77edcaae0281e4687a68f0bb08d04694b62c7b8d Mon Sep 17 00:00:00 2001 From: Alexander Nikolaev Date: Thu, 28 May 2020 18:55:13 +0200 Subject: [PATCH 13/30] CustomHeaderSessionAffinityProvider treats multiple key header values as a key extraction failure --- .../CustomHeaderSessionAffinityProvider.cs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs index 242b95bd1..8ec9dc2d7 100644 --- a/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs @@ -31,8 +31,20 @@ protected override (string Key, bool ExtractedSuccessfully) GetRequestAffinityKe { var customHeaderName = GetSettingValue(CustomHeaderNameKey, options); var keyHeaderValues = context.Request.Headers[customHeaderName]; - var encryptedRequestKey = !StringValues.IsNullOrEmpty(keyHeaderValues) ? keyHeaderValues[0] : null; // We always take the first value of a custom header storing an affinity key - return Unprotect(encryptedRequestKey); + + if (StringValues.IsNullOrEmpty(keyHeaderValues)) + { + // It means affinity key is not defined that is a successful case + return (Key: null, ExtractedSuccessfully: true); + } + + if (keyHeaderValues.Count > 1) + { + // Multiple values is an ambiguous case which is considered a key extraction failure + return (Key: null, ExtractedSuccessfully: false); + } + + return Unprotect(keyHeaderValues[0]); } protected override void SetAffinityKey(HttpContext context, in BackendConfig.BackendSessionAffinityOptions options, string unencryptedKey) From 9668cfba72be2da612b0abb550957a81bdd4b7f1 Mon Sep 17 00:00:00 2001 From: Alexander Nikolaev Date: Mon, 1 Jun 2020 13:25:36 +0200 Subject: [PATCH 14/30] Failing tests fixed --- .../Service/Config/DynamicConfigBuilderTests.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/ReverseProxy.Tests/Service/Config/DynamicConfigBuilderTests.cs b/test/ReverseProxy.Tests/Service/Config/DynamicConfigBuilderTests.cs index ccb17d4c8..67c9b9a84 100644 --- a/test/ReverseProxy.Tests/Service/Config/DynamicConfigBuilderTests.cs +++ b/test/ReverseProxy.Tests/Service/Config/DynamicConfigBuilderTests.cs @@ -27,6 +27,8 @@ private IDynamicConfigBuilder CreateConfigBuilder(IBackendsRepo backends, IRoute servicesBuilder.AddSingleton(backends); servicesBuilder.AddSingleton(routes); servicesBuilder.AddSingleton(); + servicesBuilder.AddDataProtection(); + servicesBuilder.AddLogging(); var services = servicesBuilder.BuildServiceProvider(); return services.GetRequiredService(); } From 48becbbe7d329ff3a86a377a34dfee18111bed42 Mon Sep 17 00:00:00 2001 From: Alexander Nikolaev Date: Mon, 1 Jun 2020 18:13:08 +0200 Subject: [PATCH 15/30] BaseSessionAffinityProvider's tests --- .../BaseSessionAffinityProvider.cs | 2 +- .../BaseSesstionAffinityProviderTest.cs | 174 +++++++++++++++++- 2 files changed, 170 insertions(+), 6 deletions(-) diff --git a/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs index 8989d990a..39b57c88e 100644 --- a/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs @@ -61,7 +61,7 @@ public virtual AffinityResult FindAffinitizedDestinations(HttpContext context, I { for (var i = 0; i < destinations.Count; i++) { - if (requestAffinityKey.Equals(GetDestinationAffinityKey(destinations[i]))) + if (requestAffinityKey.Key.Equals(GetDestinationAffinityKey(destinations[i]))) { // It's allowed to affinitize a request to a pool of destinations so as to enable load-balancing among them. // However, we currently stop after the first match found to avoid performance degradation. diff --git a/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTest.cs b/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTest.cs index 3bf7fd4b3..95e9b5c89 100644 --- a/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTest.cs +++ b/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTest.cs @@ -1,26 +1,182 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; using Microsoft.AspNetCore.DataProtection; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; using Microsoft.ReverseProxy.RuntimeModel; +using Moq; using Tests.Common; +using Xunit; namespace Microsoft.ReverseProxy.Service.SessionAffinity { public class BaseSesstionAffinityProviderTest : TestAutoMockBase { - private class SessionAffinityProviderStub : BaseSessionAffinityProvider + private const string InvalidKeyNull = "!invalid key - null!"; + private const string InvalidKeyThrow = "!invalid key - throw!"; + private const string KeyName = "StubAffinityKey"; + private static readonly BackendConfig.BackendSessionAffinityOptions _defaultOptions = new BackendConfig.BackendSessionAffinityOptions(true, "Stub", "Return503", new Dictionary { { "AffinityKeyName", KeyName } }); + private static readonly byte[] _encryptedNullBytes = Encoding.UTF8.GetBytes(InvalidKeyNull); + private static readonly byte[] _encryptedThrowBytes = Encoding.UTF8.GetBytes(InvalidKeyThrow); + + [Theory] + [MemberData(nameof(FindAffinitizedDestinationsCases))] + public void Request_FindAffinitizedDestinations( + HttpContext context, + DestinationInfo[] allDestinations, + AffinityStatus expectedStatus, + DestinationInfo expectedDestination, + byte[] expectedEncryptedKey, + bool unprotectCalled) + { + var dataProtector = GetDataProtector(); + var provider = new ProviderStub(dataProtector.Object, Mock().Object); + var affinityResult = provider.FindAffinitizedDestinations(context, allDestinations, "backend-1", _defaultOptions); + + if(unprotectCalled) + { + dataProtector.Verify(p => p.Unprotect(It.Is(b => b.SequenceEqual(expectedEncryptedKey))), Times.Once); + } + + Assert.Equal(expectedStatus, affinityResult.Status); + Assert.Equal(expectedDestination, affinityResult.Destinations?.FirstOrDefault()); + if (expectedDestination != null) + { + Assert.Equal(1, affinityResult.Destinations.Count); + } + else + { + Assert.Null(affinityResult.Destinations); + } + } + + [Fact] + public void FindAffinitizedDestination_AffinityDisabledOnBackend_ReturnsAffinityDisabled() + { + var provider = new ProviderStub(GetDataProtector().Object, Mock().Object); + var options = new BackendConfig.BackendSessionAffinityOptions(false, _defaultOptions.Mode, _defaultOptions.AffinityFailurePolicy, _defaultOptions.Settings); + var affinityResult = provider.FindAffinitizedDestinations(new DefaultHttpContext(), new[] { new DestinationInfo("1") }, "backend-1", options); + Assert.Equal(AffinityStatus.AffinityDisabled, affinityResult.Status); + Assert.Null(affinityResult.Destinations); + } + + [Fact] + public void AffinitizeRequest_AffinitiDisabled_DoNothing() + { + var dataProtector = GetDataProtector(); + var provider = new ProviderStub(dataProtector.Object, Mock().Object); + provider.AffinitizeRequest(new DefaultHttpContext(), default, new DestinationInfo("id")); + Assert.Null(provider.LastSetEncryptedKey); + dataProtector.Verify(p => p.Protect(It.IsAny()), Times.Never); + } + + [Fact] + public void AffinitizeRequest_RequestIsAffinitized_DoNothing() + { + var dataProtector = GetDataProtector(); + var provider = new ProviderStub(dataProtector.Object, Mock().Object); + var context = new DefaultHttpContext(); + provider.DirectlySetExtractedKeyOnContext(context, "ExtractedKey"); + provider.AffinitizeRequest(context, _defaultOptions, new DestinationInfo("id")); + Assert.Null(provider.LastSetEncryptedKey); + dataProtector.Verify(p => p.Protect(It.IsAny()), Times.Never); + } + + [Fact] + public void AffinitizeRequest_RequestIsNotAffinitized_SetAffinityKey() + { + var dataProtector = GetDataProtector(); + var provider = new ProviderStub(dataProtector.Object, Mock().Object); + var destination = new DestinationInfo("dest-A"); + provider.AffinitizeRequest(new DefaultHttpContext(), _defaultOptions, destination); + Assert.Equal("ZGVzdC1B", provider.LastSetEncryptedKey); + var keyBytes = Encoding.UTF8.GetBytes(destination.DestinationId); + dataProtector.Verify(p => p.Protect(It.Is(b => b.SequenceEqual(keyBytes))), Times.Once); + } + + [Fact] + public void FindAffinitizedDestinations_AffinityOptionSettingNotFound_Throw() + { + var provider = new ProviderStub(GetDataProtector().Object, Mock().Object); + var options = GetOptionsWithUnknownSetting(); + Assert.Throws(() => provider.FindAffinitizedDestinations(new DefaultHttpContext(), new[] { new DestinationInfo("dest-A") }, "backend-1", options)); + } + + [Fact] + public void AffinitizeRequest_AffinityOptionSettingNotFound_Throw() { - public static readonly string AffinityKeyItemName = "StubAffinityKey"; + var provider = new ProviderStub(GetDataProtector().Object, Mock().Object); + var options = GetOptionsWithUnknownSetting(); + Assert.Throws(() => provider.AffinitizeRequest(new DefaultHttpContext(), options, new DestinationInfo("dest-A"))); + } - public SessionAffinityProviderStub(IDataProtectionProvider dataProtectionProvider, ILogger logger) + [Fact] + public void Ctor_MandatoryArgumentIsNull_Throw() + { + Assert.Throws(() => new ProviderStub(null, Mock().Object)); + // CreateDataProtector will return null + Assert.Throws(() => new ProviderStub(Mock().Object, Mock().Object)); + Assert.Throws(() => new ProviderStub(GetDataProtector().Object, null)); + } + + public static IEnumerable FindAffinitizedDestinationsCases() + { + var destinations = new[] { new DestinationInfo("dest-A"), new DestinationInfo("dest-B"), new DestinationInfo("dest-C") }; + yield return new object[] { GetHttpContext(new[] { ("SomeKey", "SomeValue") }), destinations, AffinityStatus.AffinityKeyNotSet, null, null, false }; + yield return new object[] { GetHttpContext(new[] { (KeyName, "dest-B") }), destinations, AffinityStatus.OK, destinations[1], Encoding.UTF8.GetBytes("dest-B"), true }; + yield return new object[] { GetHttpContext(new[] { (KeyName, "dest-Z") }), destinations, AffinityStatus.DestinationNotFound, null, Encoding.UTF8.GetBytes("dest-Z"), true }; + yield return new object[] { GetHttpContext(new[] { (KeyName, "dest-B") }), new DestinationInfo[0], AffinityStatus.DestinationNotFound, null, Encoding.UTF8.GetBytes("dest-B"), true }; + yield return new object[] { GetHttpContext(new[] { (KeyName, InvalidKeyNull) }), destinations, AffinityStatus.AffinityKeyExtractionFailed, null, _encryptedNullBytes, true }; + yield return new object[] { GetHttpContext(new[] { (KeyName, InvalidKeyThrow) }), destinations, AffinityStatus.AffinityKeyExtractionFailed, null, _encryptedThrowBytes, true }; + } + + private static BackendConfig.BackendSessionAffinityOptions GetOptionsWithUnknownSetting() + { + return new BackendConfig.BackendSessionAffinityOptions(true, _defaultOptions.Mode, _defaultOptions.AffinityFailurePolicy, new Dictionary { { "Unknown", "ZZZ" } }); + } + + private static HttpContext GetHttpContext((string Key, string Value)[] items) + { + var context = new DefaultHttpContext + { + Items = items.ToDictionary(i => (object)i.Key, i => (object)Convert.ToBase64String(Encoding.UTF8.GetBytes(i.Value))) + }; + return context; + } + + private Mock GetDataProtector() + { + var result = Mock(); + result.Setup(p => p.Protect(It.IsAny())).Returns((byte[] k) => k); + result.Setup(p => p.Unprotect(It.IsAny())).Returns((byte[] k) => k); + result.Setup(p => p.Unprotect(It.Is(b => b.SequenceEqual(_encryptedNullBytes)))).Returns((byte[])null); + result.Setup(p => p.Unprotect(It.Is(b => b.SequenceEqual(_encryptedThrowBytes)))).Throws(); + result.Setup(p => p.CreateProtector(It.IsAny())).Returns(result.Object); + return result; + } + + private class ProviderStub : BaseSessionAffinityProvider + { + public static readonly string KeyNameSetting = "AffinityKeyName"; + + public ProviderStub(IDataProtectionProvider dataProtectionProvider, ILogger logger) : base(dataProtectionProvider, logger) {} public override string Mode => "Stub"; + public string LastSetEncryptedKey { get; private set; } + + public void DirectlySetExtractedKeyOnContext(HttpContext context, string key) + { + context.Items[AffinityKeyId] = key; + } + protected override string GetDestinationAffinityKey(DestinationInfo destination) { return destination.DestinationId; @@ -28,12 +184,20 @@ protected override string GetDestinationAffinityKey(DestinationInfo destination) protected override (string Key, bool ExtractedSuccessfully) GetRequestAffinityKey(HttpContext context, in BackendConfig.BackendSessionAffinityOptions options) { - return (Key: (string)context.Items[AffinityKeyItemName], ExtractedSuccessfully: true); + Assert.Equal(Mode, options.Mode); + var keyName = GetSettingValue(KeyNameSetting, options); + // HttpContext.Items is used here to store the request affinity key for simplicity. + // In real world scenario, a provider will extract it from request (e.g. header, cookie, etc.) + var encryptedKey = context.Items.TryGetValue(keyName, out var requestKey) ? requestKey : null; + return Unprotect((string)encryptedKey); } protected override void SetAffinityKey(HttpContext context, in BackendConfig.BackendSessionAffinityOptions options, string unencryptedKey) { - context.Items[AffinityKeyItemName] = unencryptedKey; + var keyName = GetSettingValue(KeyNameSetting, options); + var encryptedKey = Protect(unencryptedKey); + context.Items[keyName] = encryptedKey; + LastSetEncryptedKey = encryptedKey; } } } From 98db6988b9995acc3edc3d37487ddfab69cbaa06 Mon Sep 17 00:00:00 2001 From: Alexander Nikolaev Date: Mon, 1 Jun 2020 19:14:33 +0200 Subject: [PATCH 16/30] CookieSessionAffinityProvider's tests --- .../BaseSesstionAffinityProviderTest.cs | 16 +-- .../CookieSessionAffinityProviderTests.cs | 100 ++++++++++++++++++ 2 files changed, 108 insertions(+), 8 deletions(-) create mode 100644 test/ReverseProxy.Tests/Service/SessionAffinity/CookieSessionAffinityProviderTests.cs diff --git a/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTest.cs b/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTest.cs index 95e9b5c89..55f452571 100644 --- a/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTest.cs +++ b/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTest.cs @@ -20,9 +20,7 @@ public class BaseSesstionAffinityProviderTest : TestAutoMockBase private const string InvalidKeyNull = "!invalid key - null!"; private const string InvalidKeyThrow = "!invalid key - throw!"; private const string KeyName = "StubAffinityKey"; - private static readonly BackendConfig.BackendSessionAffinityOptions _defaultOptions = new BackendConfig.BackendSessionAffinityOptions(true, "Stub", "Return503", new Dictionary { { "AffinityKeyName", KeyName } }); - private static readonly byte[] _encryptedNullBytes = Encoding.UTF8.GetBytes(InvalidKeyNull); - private static readonly byte[] _encryptedThrowBytes = Encoding.UTF8.GetBytes(InvalidKeyThrow); + private readonly BackendConfig.BackendSessionAffinityOptions _defaultOptions = new BackendConfig.BackendSessionAffinityOptions(true, "Stub", "Return503", new Dictionary { { "AffinityKeyName", KeyName } }); [Theory] [MemberData(nameof(FindAffinitizedDestinationsCases))] @@ -131,13 +129,13 @@ public static IEnumerable FindAffinitizedDestinationsCases() yield return new object[] { GetHttpContext(new[] { (KeyName, "dest-B") }), destinations, AffinityStatus.OK, destinations[1], Encoding.UTF8.GetBytes("dest-B"), true }; yield return new object[] { GetHttpContext(new[] { (KeyName, "dest-Z") }), destinations, AffinityStatus.DestinationNotFound, null, Encoding.UTF8.GetBytes("dest-Z"), true }; yield return new object[] { GetHttpContext(new[] { (KeyName, "dest-B") }), new DestinationInfo[0], AffinityStatus.DestinationNotFound, null, Encoding.UTF8.GetBytes("dest-B"), true }; - yield return new object[] { GetHttpContext(new[] { (KeyName, InvalidKeyNull) }), destinations, AffinityStatus.AffinityKeyExtractionFailed, null, _encryptedNullBytes, true }; - yield return new object[] { GetHttpContext(new[] { (KeyName, InvalidKeyThrow) }), destinations, AffinityStatus.AffinityKeyExtractionFailed, null, _encryptedThrowBytes, true }; + yield return new object[] { GetHttpContext(new[] { (KeyName, InvalidKeyNull) }), destinations, AffinityStatus.AffinityKeyExtractionFailed, null, Encoding.UTF8.GetBytes(InvalidKeyNull), true }; + yield return new object[] { GetHttpContext(new[] { (KeyName, InvalidKeyThrow) }), destinations, AffinityStatus.AffinityKeyExtractionFailed, null, Encoding.UTF8.GetBytes(InvalidKeyThrow), true }; } private static BackendConfig.BackendSessionAffinityOptions GetOptionsWithUnknownSetting() { - return new BackendConfig.BackendSessionAffinityOptions(true, _defaultOptions.Mode, _defaultOptions.AffinityFailurePolicy, new Dictionary { { "Unknown", "ZZZ" } }); + return new BackendConfig.BackendSessionAffinityOptions(true, "Stub", "Return503", new Dictionary { { "Unknown", "ZZZ" } }); } private static HttpContext GetHttpContext((string Key, string Value)[] items) @@ -152,10 +150,12 @@ private static HttpContext GetHttpContext((string Key, string Value)[] items) private Mock GetDataProtector() { var result = Mock(); + var nullBytes = Encoding.UTF8.GetBytes(InvalidKeyNull); + var throwBytes = Encoding.UTF8.GetBytes(InvalidKeyThrow); result.Setup(p => p.Protect(It.IsAny())).Returns((byte[] k) => k); result.Setup(p => p.Unprotect(It.IsAny())).Returns((byte[] k) => k); - result.Setup(p => p.Unprotect(It.Is(b => b.SequenceEqual(_encryptedNullBytes)))).Returns((byte[])null); - result.Setup(p => p.Unprotect(It.Is(b => b.SequenceEqual(_encryptedThrowBytes)))).Throws(); + result.Setup(p => p.Unprotect(It.Is(b => b.SequenceEqual(nullBytes)))).Returns((byte[])null); + result.Setup(p => p.Unprotect(It.Is(b => b.SequenceEqual(throwBytes)))).Throws(); result.Setup(p => p.CreateProtector(It.IsAny())).Returns(result.Object); return result; } diff --git a/test/ReverseProxy.Tests/Service/SessionAffinity/CookieSessionAffinityProviderTests.cs b/test/ReverseProxy.Tests/Service/SessionAffinity/CookieSessionAffinityProviderTests.cs new file mode 100644 index 000000000..3f9160d10 --- /dev/null +++ b/test/ReverseProxy.Tests/Service/SessionAffinity/CookieSessionAffinityProviderTests.cs @@ -0,0 +1,100 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Text; +using Microsoft.AspNetCore.DataProtection; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract; +using Microsoft.ReverseProxy.RuntimeModel; +using Moq; +using Xunit; + +namespace Microsoft.ReverseProxy.Service.SessionAffinity +{ + public class CookieSessionAffinityProviderTests + { + private readonly CookieSessionAffinityProviderOptions _defaultProviderOptions = new CookieSessionAffinityProviderOptions(); + private readonly BackendConfig.BackendSessionAffinityOptions _defaultOptions = new BackendConfig.BackendSessionAffinityOptions(true, "Cookie", "Return503", null); + private readonly IReadOnlyList _destinations = new[] { new DestinationInfo("dest-A"), new DestinationInfo("dest-B"), new DestinationInfo("dest-C") }; + + [Fact] + public void FindAffinitizedDestination_AffinityKeyIsNotSetOnRequest_ReturnKeyNotSet() + { + var provider = new CookieSessionAffinityProvider(Options.Create(_defaultProviderOptions), GetDataProtector().Object, GetLogger().Object); + var context = new DefaultHttpContext(); + context.Request.Headers["Cookie"] = new[] { $"Some-Cookie=ZZZ" }; + + var affinityResult = provider.FindAffinitizedDestinations(context, _destinations, "backend-1", _defaultOptions); + + Assert.Equal(AffinityStatus.AffinityKeyNotSet, affinityResult.Status); + Assert.Null(affinityResult.Destinations); + } + + [Fact] + public void FindAffinitizedDestination_AffinityKeyIsSetOnRequest_Success() + { + var provider = new CookieSessionAffinityProvider(Options.Create(_defaultProviderOptions), GetDataProtector().Object, GetLogger().Object); + var context = new DefaultHttpContext(); + var affinitizedDestination = _destinations[1]; + context.Request.Headers["Cookie"] = GetCookieWithAffinity(affinitizedDestination); + + var affinityResult = provider.FindAffinitizedDestinations(context, _destinations, "backend-1", _defaultOptions); + + Assert.Equal(AffinityStatus.OK, affinityResult.Status); + Assert.Equal(1, affinityResult.Destinations.Count); + Assert.Equal(affinitizedDestination, affinityResult.Destinations[0]); + } + + [Fact] + public void AffinitizedRequest_AffinityKeyIsNotExtracted_SetKeyOnResponse() + { + var provider = new CookieSessionAffinityProvider(Options.Create(_defaultProviderOptions), GetDataProtector().Object, GetLogger().Object); + var context = new DefaultHttpContext(); + + provider.AffinitizeRequest(context, _defaultOptions, _destinations[1]); + + var affinityCookieHeader = context.Response.Headers["Set-Cookie"]; + Assert.Equal(".Microsoft.ReverseProxy.Affinity=ZGVzdC1C; path=/; httponly", affinityCookieHeader); + } + + [Fact] + public void AffinitizedRequest_AffinityKeyIsExtracted_DoNothing() + { + var provider = new CookieSessionAffinityProvider(Options.Create(_defaultProviderOptions), GetDataProtector().Object, GetLogger().Object); + var context = new DefaultHttpContext(); + var affinitizedDestination = _destinations[0]; + context.Request.Headers["Cookie"] = GetCookieWithAffinity(affinitizedDestination); + + var affinityResult = provider.FindAffinitizedDestinations(context, _destinations, "backend-1", _defaultOptions); + + Assert.Equal(AffinityStatus.OK, affinityResult.Status); + + provider.AffinitizeRequest(context, _defaultOptions, affinitizedDestination); + + Assert.False(context.Response.Headers.ContainsKey("Cookie")); + } + + private string[] GetCookieWithAffinity(DestinationInfo affinitizedDestination) + { + return new[] { $"Some-Cookie=ZZZ", $"{_defaultProviderOptions.Cookie.Name}={Convert.ToBase64String(Encoding.UTF8.GetBytes(affinitizedDestination.DestinationId))}" }; + } + + private static Mock> GetLogger() + { + return new Mock>(); + } + + private static Mock GetDataProtector() + { + var protector = new Mock(); + protector.Setup(p => p.CreateProtector(It.IsAny())).Returns(protector.Object); + protector.Setup(p => p.Protect(It.IsAny())).Returns((byte[] b) => b); + protector.Setup(p => p.Unprotect(It.IsAny())).Returns((byte[] b) => b); + return protector; + } + } +} From 1fb3e1676d2418deceb77b765c5d5f4a0e044221 Mon Sep 17 00:00:00 2001 From: Alexander Nikolaev Date: Tue, 2 Jun 2020 14:24:45 +0200 Subject: [PATCH 17/30] - CustomHeaderSessionAffinityProvider's tests - Failure policies tests - Naming fixes --- .../Contract/SessionAffinityBuiltIns.cs | 2 +- .../CustomHeaderSessionAffinityProvider.cs | 2 +- ...s => RedistributeAffinityFailurePolicy.cs} | 0 ...=> Return503ErrorAffinityFailurePolicy.cs} | 10 +- .../SessionAffinity/AffinityTestHelper.cs | 33 +++++++ ...s => BaseSesstionAffinityProviderTests.cs} | 2 +- .../CookieSessionAffinityProviderTests.cs | 67 ++++++++----- ...ustomHeaderSessionAffinityProviderTests.cs | 95 +++++++++++++++++++ .../RedistributeAffinityFailurePolicyTests.cs | 35 +++++++ ...eturn503ErrorAffinityFailurePolicyTests.cs | 40 ++++++++ 10 files changed, 259 insertions(+), 27 deletions(-) rename src/ReverseProxy/Service/SessionAffinity/{PickRandomMissingDestinationHandler.cs => RedistributeAffinityFailurePolicy.cs} (100%) rename src/ReverseProxy/Service/SessionAffinity/{ReturnErrorMissingDestinationHandler.cs => Return503ErrorAffinityFailurePolicy.cs} (63%) create mode 100644 test/ReverseProxy.Tests/Service/SessionAffinity/AffinityTestHelper.cs rename test/ReverseProxy.Tests/Service/SessionAffinity/{BaseSesstionAffinityProviderTest.cs => BaseSesstionAffinityProviderTests.cs} (99%) create mode 100644 test/ReverseProxy.Tests/Service/SessionAffinity/CustomHeaderSessionAffinityProviderTests.cs create mode 100644 test/ReverseProxy.Tests/Service/SessionAffinity/RedistributeAffinityFailurePolicyTests.cs create mode 100644 test/ReverseProxy.Tests/Service/SessionAffinity/Return503ErrorAffinityFailurePolicyTests.cs diff --git a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityBuiltIns.cs b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityBuiltIns.cs index 4df825862..347d5aa3e 100644 --- a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityBuiltIns.cs +++ b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityBuiltIns.cs @@ -12,7 +12,7 @@ public static class Modes { public static string Cookie => "Cookie"; - public static string CustomHeander => "CustomHeader"; + public static string CustomHeader => "CustomHeader"; } public static class AffinityFailurePolicies diff --git a/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs index 8ec9dc2d7..d7df833a8 100644 --- a/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs @@ -20,7 +20,7 @@ public CustomHeaderSessionAffinityProvider( : base(dataProtectionProvider, logger) {} - public override string Mode => SessionAffinityBuiltIns.Modes.CustomHeander; + public override string Mode => SessionAffinityBuiltIns.Modes.CustomHeader; protected override string GetDestinationAffinityKey(DestinationInfo destination) { diff --git a/src/ReverseProxy/Service/SessionAffinity/PickRandomMissingDestinationHandler.cs b/src/ReverseProxy/Service/SessionAffinity/RedistributeAffinityFailurePolicy.cs similarity index 100% rename from src/ReverseProxy/Service/SessionAffinity/PickRandomMissingDestinationHandler.cs rename to src/ReverseProxy/Service/SessionAffinity/RedistributeAffinityFailurePolicy.cs diff --git a/src/ReverseProxy/Service/SessionAffinity/ReturnErrorMissingDestinationHandler.cs b/src/ReverseProxy/Service/SessionAffinity/Return503ErrorAffinityFailurePolicy.cs similarity index 63% rename from src/ReverseProxy/Service/SessionAffinity/ReturnErrorMissingDestinationHandler.cs rename to src/ReverseProxy/Service/SessionAffinity/Return503ErrorAffinityFailurePolicy.cs index e7b1fa2af..4b73bc02f 100644 --- a/src/ReverseProxy/Service/SessionAffinity/ReturnErrorMissingDestinationHandler.cs +++ b/src/ReverseProxy/Service/SessionAffinity/Return503ErrorAffinityFailurePolicy.cs @@ -14,8 +14,16 @@ internal class Return503ErrorAffinityFailurePolicy : IAffinityFailurePolicy public Task Handle(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, AffinityStatus affinityStatus) { + if (affinityStatus == AffinityStatus.OK + || affinityStatus == AffinityStatus.AffinityKeyNotSet + || affinityStatus == AffinityStatus.AffinityDisabled) + { + // We shouldn't get here, but allow the request to proceed further if that's the case. + return Task.FromResult(true); + } + context.Response.StatusCode = 503; - return Task.FromResult(true); + return Task.FromResult(false); } } } diff --git a/test/ReverseProxy.Tests/Service/SessionAffinity/AffinityTestHelper.cs b/test/ReverseProxy.Tests/Service/SessionAffinity/AffinityTestHelper.cs new file mode 100644 index 000000000..02cb3879b --- /dev/null +++ b/test/ReverseProxy.Tests/Service/SessionAffinity/AffinityTestHelper.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Text; +using Microsoft.AspNetCore.DataProtection; +using Microsoft.Extensions.Logging; +using Moq; + +namespace Microsoft.ReverseProxy.Service.SessionAffinity +{ + public static class AffinityTestHelper + { + public static Mock> GetLogger() + { + return new Mock>(); + } + + public static Mock GetDataProtector() + { + var protector = new Mock(); + protector.Setup(p => p.CreateProtector(It.IsAny())).Returns(protector.Object); + protector.Setup(p => p.Protect(It.IsAny())).Returns((byte[] b) => b); + protector.Setup(p => p.Unprotect(It.IsAny())).Returns((byte[] b) => b); + return protector; + } + + public static string ToUTF8BytesInBase64(this string text) + { + return Convert.ToBase64String(Encoding.UTF8.GetBytes(text)); + } + } +} diff --git a/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTest.cs b/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTests.cs similarity index 99% rename from test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTest.cs rename to test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTests.cs index 55f452571..d93566d8f 100644 --- a/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTest.cs +++ b/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTests.cs @@ -15,7 +15,7 @@ namespace Microsoft.ReverseProxy.Service.SessionAffinity { - public class BaseSesstionAffinityProviderTest : TestAutoMockBase + public class BaseSesstionAffinityProviderTests : TestAutoMockBase { private const string InvalidKeyNull = "!invalid key - null!"; private const string InvalidKeyThrow = "!invalid key - throw!"; diff --git a/test/ReverseProxy.Tests/Service/SessionAffinity/CookieSessionAffinityProviderTests.cs b/test/ReverseProxy.Tests/Service/SessionAffinity/CookieSessionAffinityProviderTests.cs index 3f9160d10..6527fc74c 100644 --- a/test/ReverseProxy.Tests/Service/SessionAffinity/CookieSessionAffinityProviderTests.cs +++ b/test/ReverseProxy.Tests/Service/SessionAffinity/CookieSessionAffinityProviderTests.cs @@ -3,14 +3,10 @@ using System; using System.Collections.Generic; -using System.Text; -using Microsoft.AspNetCore.DataProtection; using Microsoft.AspNetCore.Http; -using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract; using Microsoft.ReverseProxy.RuntimeModel; -using Moq; using Xunit; namespace Microsoft.ReverseProxy.Service.SessionAffinity @@ -24,7 +20,13 @@ public class CookieSessionAffinityProviderTests [Fact] public void FindAffinitizedDestination_AffinityKeyIsNotSetOnRequest_ReturnKeyNotSet() { - var provider = new CookieSessionAffinityProvider(Options.Create(_defaultProviderOptions), GetDataProtector().Object, GetLogger().Object); + var provider = new CookieSessionAffinityProvider( + Options.Create(_defaultProviderOptions), + AffinityTestHelper.GetDataProtector().Object, + AffinityTestHelper.GetLogger().Object); + + Assert.Equal(SessionAffinityBuiltIns.Modes.Cookie, provider.Mode); + var context = new DefaultHttpContext(); context.Request.Headers["Cookie"] = new[] { $"Some-Cookie=ZZZ" }; @@ -37,7 +39,10 @@ public void FindAffinitizedDestination_AffinityKeyIsNotSetOnRequest_ReturnKeyNot [Fact] public void FindAffinitizedDestination_AffinityKeyIsSetOnRequest_Success() { - var provider = new CookieSessionAffinityProvider(Options.Create(_defaultProviderOptions), GetDataProtector().Object, GetLogger().Object); + var provider = new CookieSessionAffinityProvider( + Options.Create(_defaultProviderOptions), + AffinityTestHelper.GetDataProtector().Object, + AffinityTestHelper.GetLogger().Object); var context = new DefaultHttpContext(); var affinitizedDestination = _destinations[1]; context.Request.Headers["Cookie"] = GetCookieWithAffinity(affinitizedDestination); @@ -52,7 +57,10 @@ public void FindAffinitizedDestination_AffinityKeyIsSetOnRequest_Success() [Fact] public void AffinitizedRequest_AffinityKeyIsNotExtracted_SetKeyOnResponse() { - var provider = new CookieSessionAffinityProvider(Options.Create(_defaultProviderOptions), GetDataProtector().Object, GetLogger().Object); + var provider = new CookieSessionAffinityProvider( + Options.Create(_defaultProviderOptions), + AffinityTestHelper.GetDataProtector().Object, + AffinityTestHelper.GetLogger().Object); var context = new DefaultHttpContext(); provider.AffinitizeRequest(context, _defaultOptions, _destinations[1]); @@ -61,10 +69,37 @@ public void AffinitizedRequest_AffinityKeyIsNotExtracted_SetKeyOnResponse() Assert.Equal(".Microsoft.ReverseProxy.Affinity=ZGVzdC1C; path=/; httponly", affinityCookieHeader); } + [Fact] + public void AffinitizeRequest_CookieBuilderSettingsChanged_UseNewSettings() + { + var providerOptions = new CookieSessionAffinityProviderOptions(); + providerOptions.Cookie.Domain = "mydomain.my"; + providerOptions.Cookie.HttpOnly = false; + providerOptions.Cookie.IsEssential = true; + providerOptions.Cookie.MaxAge = TimeSpan.FromHours(1); + providerOptions.Cookie.Name = "My.Affinity"; + providerOptions.Cookie.Path = "/some"; + providerOptions.Cookie.SameSite = SameSiteMode.Lax; + providerOptions.Cookie.SecurePolicy = CookieSecurePolicy.Always; + var provider = new CookieSessionAffinityProvider( + Options.Create(providerOptions), + AffinityTestHelper.GetDataProtector().Object, + AffinityTestHelper.GetLogger().Object); + var context = new DefaultHttpContext(); + + provider.AffinitizeRequest(context, _defaultOptions, _destinations[1]); + + var affinityCookieHeader = context.Response.Headers["Set-Cookie"]; + Assert.Equal("My.Affinity=ZGVzdC1C; max-age=3600; domain=mydomain.my; path=/some; secure; samesite=lax", affinityCookieHeader); + } + [Fact] public void AffinitizedRequest_AffinityKeyIsExtracted_DoNothing() { - var provider = new CookieSessionAffinityProvider(Options.Create(_defaultProviderOptions), GetDataProtector().Object, GetLogger().Object); + var provider = new CookieSessionAffinityProvider( + Options.Create(_defaultProviderOptions), + AffinityTestHelper.GetDataProtector().Object, + AffinityTestHelper.GetLogger().Object); var context = new DefaultHttpContext(); var affinitizedDestination = _destinations[0]; context.Request.Headers["Cookie"] = GetCookieWithAffinity(affinitizedDestination); @@ -80,21 +115,7 @@ public void AffinitizedRequest_AffinityKeyIsExtracted_DoNothing() private string[] GetCookieWithAffinity(DestinationInfo affinitizedDestination) { - return new[] { $"Some-Cookie=ZZZ", $"{_defaultProviderOptions.Cookie.Name}={Convert.ToBase64String(Encoding.UTF8.GetBytes(affinitizedDestination.DestinationId))}" }; - } - - private static Mock> GetLogger() - { - return new Mock>(); - } - - private static Mock GetDataProtector() - { - var protector = new Mock(); - protector.Setup(p => p.CreateProtector(It.IsAny())).Returns(protector.Object); - protector.Setup(p => p.Protect(It.IsAny())).Returns((byte[] b) => b); - protector.Setup(p => p.Unprotect(It.IsAny())).Returns((byte[] b) => b); - return protector; + return new[] { $"Some-Cookie=ZZZ", $"{_defaultProviderOptions.Cookie.Name}={affinitizedDestination.DestinationId.ToUTF8BytesInBase64()}" }; } } } diff --git a/test/ReverseProxy.Tests/Service/SessionAffinity/CustomHeaderSessionAffinityProviderTests.cs b/test/ReverseProxy.Tests/Service/SessionAffinity/CustomHeaderSessionAffinityProviderTests.cs new file mode 100644 index 000000000..d30481d50 --- /dev/null +++ b/test/ReverseProxy.Tests/Service/SessionAffinity/CustomHeaderSessionAffinityProviderTests.cs @@ -0,0 +1,95 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using Microsoft.AspNetCore.Http; +using Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract; +using Microsoft.ReverseProxy.RuntimeModel; +using Xunit; + +namespace Microsoft.ReverseProxy.Service.SessionAffinity +{ + public class CustomHeaderSessionAffinityProviderTests + { + private const string AffinityHeaderName = "X-MyAffinity"; + private readonly BackendConfig.BackendSessionAffinityOptions _defaultOptions = + new BackendConfig.BackendSessionAffinityOptions(true, "Cookie", "Return503", new Dictionary { { "CustomHeaderName", AffinityHeaderName } }); + private readonly IReadOnlyList _destinations = new[] { new DestinationInfo("dest-A"), new DestinationInfo("dest-B"), new DestinationInfo("dest-C") }; + + [Fact] + public void FindAffinitizedDestination_AffinityKeyIsNotSetOnRequest_ReturnKeyNotSet() + { + var provider = new CustomHeaderSessionAffinityProvider(AffinityTestHelper.GetDataProtector().Object, AffinityTestHelper.GetLogger().Object); + + Assert.Equal(SessionAffinityBuiltIns.Modes.CustomHeader, provider.Mode); + + var context = new DefaultHttpContext(); + context.Request.Headers["SomeHeader"] = new[] { "SomeValue" }; + + var affinityResult = provider.FindAffinitizedDestinations(context, _destinations, "backend-1", _defaultOptions); + + Assert.Equal(AffinityStatus.AffinityKeyNotSet, affinityResult.Status); + Assert.Null(affinityResult.Destinations); + } + + [Fact] + public void FindAffinitizedDestination_AffinityKeyIsSetOnRequest_Success() + { + var provider = new CustomHeaderSessionAffinityProvider(AffinityTestHelper.GetDataProtector().Object, AffinityTestHelper.GetLogger().Object); + var context = new DefaultHttpContext(); + context.Request.Headers["SomeHeader"] = new[] { "SomeValue" }; + var affinitizedDestination = _destinations[1]; + context.Request.Headers[AffinityHeaderName] = new[] { affinitizedDestination.DestinationId.ToUTF8BytesInBase64() }; + + var affinityResult = provider.FindAffinitizedDestinations(context, _destinations, "backend-1", _defaultOptions); + + Assert.Equal(AffinityStatus.OK, affinityResult.Status); + Assert.Equal(1, affinityResult.Destinations.Count); + Assert.Equal(affinitizedDestination, affinityResult.Destinations[0]); + } + + [Fact] + public void FindAffinitizedDestination_CustomHeaderNameIsNotSpecified_Throw() + { + var options = new BackendConfig.BackendSessionAffinityOptions(true, "CustomHeader", "Return503", null); + var provider = new CustomHeaderSessionAffinityProvider(AffinityTestHelper.GetDataProtector().Object, AffinityTestHelper.GetLogger().Object); + var context = new DefaultHttpContext(); + context.Request.Headers["SomeHeader"] = new[] { "SomeValue" }; + + Assert.Throws(() => provider.FindAffinitizedDestinations(context, _destinations, "backend-1", options)); + } + + [Fact] + public void AffinitizedRequest_AffinityKeyIsNotExtracted_SetKeyOnResponse() + { + var provider = new CustomHeaderSessionAffinityProvider(AffinityTestHelper.GetDataProtector().Object, AffinityTestHelper.GetLogger().Object); + var context = new DefaultHttpContext(); + var chosenDestination = _destinations[1]; + var expectedAffinityHeaderValue = chosenDestination.DestinationId.ToUTF8BytesInBase64(); + + provider.AffinitizeRequest(context, _defaultOptions, chosenDestination); + + Assert.True(context.Response.Headers.ContainsKey(AffinityHeaderName)); + Assert.Equal(expectedAffinityHeaderValue, context.Response.Headers[AffinityHeaderName]); + } + + [Fact] + public void AffinitizedRequest_AffinityKeyIsExtracted_DoNothing() + { + var provider = new CustomHeaderSessionAffinityProvider(AffinityTestHelper.GetDataProtector().Object, AffinityTestHelper.GetLogger().Object); + var context = new DefaultHttpContext(); + context.Request.Headers["SomeHeader"] = new[] { "SomeValue" }; + var affinitizedDestination = _destinations[1]; + context.Request.Headers[AffinityHeaderName] = new[] { affinitizedDestination.DestinationId.ToUTF8BytesInBase64() }; + + var affinityResult = provider.FindAffinitizedDestinations(context, _destinations, "backend-1", _defaultOptions); + + Assert.Equal(AffinityStatus.OK, affinityResult.Status); + + provider.AffinitizeRequest(context, _defaultOptions, affinitizedDestination); + + Assert.False(context.Response.Headers.ContainsKey(AffinityHeaderName)); + } + } +} diff --git a/test/ReverseProxy.Tests/Service/SessionAffinity/RedistributeAffinityFailurePolicyTests.cs b/test/ReverseProxy.Tests/Service/SessionAffinity/RedistributeAffinityFailurePolicyTests.cs new file mode 100644 index 000000000..35d5eec73 --- /dev/null +++ b/test/ReverseProxy.Tests/Service/SessionAffinity/RedistributeAffinityFailurePolicyTests.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract; +using Xunit; + +namespace Microsoft.ReverseProxy.Service.SessionAffinity +{ + public class RedistributeAffinityFailurePolicyTests + { + [Theory] + [MemberData(nameof(HandleCases))] + public async Task Handle_AnyAffinitStatus_ReturnTrue(AffinityStatus status) + { + var policy = new RedistributeAffinityFailurePolicy(); + + Assert.Equal(SessionAffinityBuiltIns.AffinityFailurePolicies.Redistribute, policy.Name); + Assert.True(await policy.Handle(new DefaultHttpContext(), default, status)); + } + + public static IEnumerable HandleCases() + { + // Successful statuses are also included because redistribute policy is not supposed to validate this parameter + // and therefore must react properly to all affinity statuses. + foreach(AffinityStatus status in Enum.GetValues(typeof(AffinityStatus))) + { + yield return new object[] { status }; + } + } + } +} diff --git a/test/ReverseProxy.Tests/Service/SessionAffinity/Return503ErrorAffinityFailurePolicyTests.cs b/test/ReverseProxy.Tests/Service/SessionAffinity/Return503ErrorAffinityFailurePolicyTests.cs new file mode 100644 index 000000000..92121fec4 --- /dev/null +++ b/test/ReverseProxy.Tests/Service/SessionAffinity/Return503ErrorAffinityFailurePolicyTests.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract; +using Xunit; + +namespace Microsoft.ReverseProxy.Service.SessionAffinity +{ + public class Return503ErrorAffinityFailurePolicyTests + { + [Theory] + [InlineData(AffinityStatus.DestinationNotFound)] + [InlineData(AffinityStatus.AffinityKeyExtractionFailed)] + public async Task Handle_FaultyAffinityStatus_RespondWith503(AffinityStatus status) + { + var policy = new Return503ErrorAffinityFailurePolicy(); + var context = new DefaultHttpContext(); + + Assert.Equal(SessionAffinityBuiltIns.AffinityFailurePolicies.Return503Error, policy.Name); + + Assert.False(await policy.Handle(context, default, status)); + Assert.Equal(503, context.Response.StatusCode); + } + + [Theory] + [InlineData(AffinityStatus.OK)] + [InlineData(AffinityStatus.AffinityKeyNotSet)] + [InlineData(AffinityStatus.AffinityDisabled)] + public async Task Handle_SuccessfulAffinityStatus_ReturnTrue(AffinityStatus status) + { + var policy = new Return503ErrorAffinityFailurePolicy(); + var context = new DefaultHttpContext(); + + Assert.True(await policy.Handle(context, default, status)); + Assert.Equal(200, context.Response.StatusCode); + } + } +} From b6a89acc3e267e1887539731444f88213a946ccd Mon Sep 17 00:00:00 2001 From: Alexander Nikolaev Date: Tue, 2 Jun 2020 18:33:59 +0200 Subject: [PATCH 18/30] - AffinitizedDestinationLookupMiddleware tests --- .../AffinitizedDestinationLookupMiddleware.cs | 19 +- .../SessionAffinityMiddlewareHelper.cs | 1 - ...nitizedDestinationLookupMiddlewareTests.cs | 168 ++++++++++++++++++ .../DestinationInitializerMiddlewareTests.cs | 2 - 4 files changed, 179 insertions(+), 11 deletions(-) create mode 100644 test/ReverseProxy.Tests/Middleware/AffinitizedDestinationLookupMiddlewareTests.cs diff --git a/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs b/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs index c1966d0be..f03d11b3c 100644 --- a/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs +++ b/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs @@ -66,17 +66,20 @@ public async Task Invoke(HttpContext context) break; case AffinityStatus.AffinityKeyExtractionFailed: case AffinityStatus.DestinationNotFound: - await _operationLogger.ExecuteAsync("ReverseProxy.HandleAffinityFailure", async () => + var failureHandled = await _operationLogger.ExecuteAsync("ReverseProxy.HandleAffinityFailure", async () => { var failurePolicy = _affinityFailurePolicies.GetRequiredServiceById(options.AffinityFailurePolicy); - if (!await failurePolicy.Handle(context, options, affinityResult.Status)) - { - // Policy reported the failure is unrecoverable and took the full responsibility for its handling, - // so we simply stop processing. - Log.AffinityResolutionFailedForBackend(_logger, backend.BackendId); - return; - } + return await failurePolicy.Handle(context, options, affinityResult.Status); }); + + if (!failureHandled) + { + // Policy reported the failure is unrecoverable and took the full responsibility for its handling, + // so we simply stop processing. + Log.AffinityResolutionFailedForBackend(_logger, backend.BackendId); + return; + } + break; default: throw new NotSupportedException($"Affinity status '{affinityResult.Status}' is not supported."); diff --git a/src/ReverseProxy/Service/SessionAffinity/SessionAffinityMiddlewareHelper.cs b/src/ReverseProxy/Service/SessionAffinity/SessionAffinityMiddlewareHelper.cs index bd73c6384..8152d53d1 100644 --- a/src/ReverseProxy/Service/SessionAffinity/SessionAffinityMiddlewareHelper.cs +++ b/src/ReverseProxy/Service/SessionAffinity/SessionAffinityMiddlewareHelper.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; -using Microsoft.ReverseProxy.Service.SessionAffinity; namespace Microsoft.ReverseProxy.Service.SessionAffinity { diff --git a/test/ReverseProxy.Tests/Middleware/AffinitizedDestinationLookupMiddlewareTests.cs b/test/ReverseProxy.Tests/Middleware/AffinitizedDestinationLookupMiddlewareTests.cs new file mode 100644 index 000000000..67fc07dd0 --- /dev/null +++ b/test/ReverseProxy.Tests/Middleware/AffinitizedDestinationLookupMiddlewareTests.cs @@ -0,0 +1,168 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using Microsoft.ReverseProxy.Abstractions.Telemetry; +using Microsoft.ReverseProxy.RuntimeModel; +using Microsoft.ReverseProxy.Service.Management; +using Microsoft.ReverseProxy.Service.Proxy.Infrastructure; +using Microsoft.ReverseProxy.Service.SessionAffinity; +using Microsoft.ReverseProxy.Signals; +using Moq; +using Xunit; + +namespace Microsoft.ReverseProxy.Middleware +{ + public class AffinitizedDestinationLookupMiddlewareTests + { + private const string AffinitizedDestinationName = "dest-B"; + private readonly IReadOnlyList _destinations = new[] { new DestinationInfo("dest-A"), new DestinationInfo(AffinitizedDestinationName), new DestinationInfo("dest-C") }; + private readonly BackendConfig _backendConfig = new BackendConfig(default, default, new BackendConfig.BackendSessionAffinityOptions(true, "Mode-B", "Policy-1", null)); + + [Theory] + [InlineData(AffinityStatus.AffinityKeyNotSet, null)] + [InlineData(AffinityStatus.AffinityDisabled, null)] + [InlineData(AffinityStatus.OK, AffinitizedDestinationName)] + public async Task Invoke_SuccessfulFlow_CallNext(AffinityStatus status, string foundDestinationId) + { + var backend = GetBackend(); + var foundDestinations = foundDestinationId != null ? _destinations.Where(d => d.DestinationId == foundDestinationId).ToArray() : null; + var invokedMode = string.Empty; + var providers = RegisterAffinityProviders( + _destinations, + backend.BackendId, + ("Mode-A", AffinityStatus.DestinationNotFound, null, p => throw new InvalidOperationException($"Provider {p.Mode} call is not expected.")), + ("Mode-B", status, foundDestinations, p => invokedMode = p.Mode)); + var middleware = new AffinitizedDestinationLookupMiddleware(c => Task.CompletedTask, + providers.Select(p => p.Object), new IAffinityFailurePolicy[0], + GetOperationLogger(false), + new Mock>().Object); + var context = new DefaultHttpContext(); + context.Features.Set(backend); + var destinationFeature = GetDestinationsFeature(_destinations); + context.Features.Set(destinationFeature); + + await middleware.Invoke(context); + + Assert.Equal("Mode-B", invokedMode); + providers[0].VerifyGet(p => p.Mode, Times.Once); + providers[0].VerifyNoOtherCalls(); + providers[1].VerifyAll(); + + if (foundDestinationId != null) + { + Assert.Equal(1, destinationFeature.Destinations.Count); + Assert.Equal(foundDestinationId, destinationFeature.Destinations[0].DestinationId); + } + else + { + Assert.Equal(_destinations, destinationFeature.Destinations); + } + } + + [Theory] + [InlineData(AffinityStatus.DestinationNotFound, true)] + [InlineData(AffinityStatus.DestinationNotFound, false)] + [InlineData(AffinityStatus.AffinityKeyExtractionFailed, true)] + [InlineData(AffinityStatus.AffinityKeyExtractionFailed, false)] + public async Task Invoke_ErrorFlow_CallFailurePolicy(AffinityStatus affinityStatus, bool handled) + { + var backend = GetBackend(); + var providers = RegisterAffinityProviders(_destinations, backend.BackendId, ("Mode-B", affinityStatus, null, _ => { })); + var invokedPolicy = string.Empty; + var failurePolicies = RegisterFailurePolicies( + affinityStatus, + ("Policy-0", false, p => throw new InvalidOperationException($"Policy {p.Name} call is not expected.")), + ("Policy-1", handled, p => invokedPolicy = p.Name)); + var nextInvoked = false; + var middleware = new AffinitizedDestinationLookupMiddleware(c => { + nextInvoked = true; + return Task.CompletedTask; + }, + providers.Select(p => p.Object), failurePolicies.Select(p => p.Object), + GetOperationLogger(true), + new Mock>().Object); + var context = new DefaultHttpContext(); + context.Features.Set(backend); + var destinationFeature = GetDestinationsFeature(_destinations); + context.Features.Set(destinationFeature); + + await middleware.Invoke(context); + + Assert.Equal("Policy-1", invokedPolicy); + Assert.Equal(handled, nextInvoked); + failurePolicies[0].VerifyGet(p => p.Name, Times.Once); + failurePolicies[0].VerifyNoOtherCalls(); + failurePolicies[1].VerifyAll(); + } + + private BackendInfo GetBackend() + { + var destinationManager = new Mock(); + destinationManager.SetupGet(m => m.Items).Returns(SignalFactory.Default.CreateSignal(_destinations)); + var backend = new BackendInfo("backend-1", destinationManager.Object, new Mock().Object); + backend.Config.Value = _backendConfig; + return backend; + } + + private IReadOnlyList> RegisterAffinityProviders( + IReadOnlyList expectedDestinations, + string expectedBackend, + params (string Mode, AffinityStatus Status, DestinationInfo[] Destinations, Action Callback)[] prototypes) + { + var result = new List>(); + foreach (var (mode, status, destinations, callback) in prototypes) + { + var provider = new Mock(MockBehavior.Strict); + provider.SetupGet(p => p.Mode).Returns(mode); + provider.Setup(p => p.FindAffinitizedDestinations( + It.IsAny(), + expectedDestinations, + expectedBackend, + _backendConfig.SessionAffinityOptions)) + .Returns(new AffinityResult(destinations, status)) + .Callback(() => callback(provider.Object)); + result.Add(provider); + } + return result.AsReadOnly(); + } + + private IReadOnlyList> RegisterFailurePolicies(AffinityStatus expectedStatus, params (string Name, bool Handled, Action Callback)[] prototypes) + { + var result = new List>(); + foreach (var (name, handled, callback) in prototypes) + { + var policy = new Mock(MockBehavior.Strict); + policy.SetupGet(p => p.Name).Returns(name); + policy.Setup(p => p.Handle(It.IsAny(), It.Is(o => o.AffinityFailurePolicy == name), expectedStatus)) + .ReturnsAsync(handled) + .Callback(() => callback(policy.Object)); + result.Add(policy); + } + return result.AsReadOnly(); + } + + private IAvailableDestinationsFeature GetDestinationsFeature(IReadOnlyList destinations) + { + var result = new Mock(MockBehavior.Strict); + result.SetupProperty(p => p.Destinations, destinations); + return result.Object; + } + + private IOperationLogger GetOperationLogger(bool callFailurePolicy) + { + var result = new Mock>(MockBehavior.Strict); + result.Setup(l => l.Execute(It.IsAny(), It.IsAny>())).Returns((string name, Func callback) => callback()); + if (callFailurePolicy) + { + result.Setup(l => l.ExecuteAsync(It.IsAny(), It.IsAny>>())).Returns(async (string name, Func> callback) => await callback()); + } + return result.Object; + } + } +} diff --git a/test/ReverseProxy.Tests/Middleware/DestinationInitializerMiddlewareTests.cs b/test/ReverseProxy.Tests/Middleware/DestinationInitializerMiddlewareTests.cs index dc01dbdb5..7c46385ad 100644 --- a/test/ReverseProxy.Tests/Middleware/DestinationInitializerMiddlewareTests.cs +++ b/test/ReverseProxy.Tests/Middleware/DestinationInitializerMiddlewareTests.cs @@ -7,8 +7,6 @@ using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Routing; using Microsoft.AspNetCore.Routing.Patterns; -using Microsoft.ReverseProxy.Abstractions.Telemetry; -using Microsoft.ReverseProxy.Telemetry; using Microsoft.ReverseProxy.RuntimeModel; using Microsoft.ReverseProxy.Service.Management; using Microsoft.ReverseProxy.Service.Proxy.Infrastructure; From 591837e64d302509bd523fe51975c4c7e3bac172 Mon Sep 17 00:00:00 2001 From: Alexander Nikolaev Date: Wed, 3 Jun 2020 13:55:32 +0200 Subject: [PATCH 19/30] Renaming --- samples/ReverseProxy.Sample/Startup.cs | 1 - ...SessionAffinityBuiltIns.cs => SessionAffinityConstants.cs} | 2 +- .../Contract/SessionAffinityDefaultOptions.cs | 4 ++-- .../Service/SessionAffinity/BaseSessionAffinityProvider.cs | 1 + .../Service/SessionAffinity/CookieSessionAffinityProvider.cs | 2 +- .../SessionAffinity/CustomHeaderSessionAffinityProvider.cs | 2 +- .../SessionAffinity/RedistributeAffinityFailurePolicy.cs | 2 +- .../SessionAffinity/Return503ErrorAffinityFailurePolicy.cs | 2 +- .../SessionAffinity/CookieSessionAffinityProviderTests.cs | 2 +- .../CustomHeaderSessionAffinityProviderTests.cs | 2 +- .../SessionAffinity/RedistributeAffinityFailurePolicyTests.cs | 2 +- .../Return503ErrorAffinityFailurePolicyTests.cs | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) rename src/ReverseProxy/Abstractions/BackendDiscovery/Contract/{SessionAffinityBuiltIns.cs => SessionAffinityConstants.cs} (92%) diff --git a/samples/ReverseProxy.Sample/Startup.cs b/samples/ReverseProxy.Sample/Startup.cs index 85eb30cfb..e5eb97b56 100644 --- a/samples/ReverseProxy.Sample/Startup.cs +++ b/samples/ReverseProxy.Sample/Startup.cs @@ -4,7 +4,6 @@ using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; -using Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract; using Microsoft.ReverseProxy.Middleware; namespace Microsoft.ReverseProxy.Sample diff --git a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityBuiltIns.cs b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityConstants.cs similarity index 92% rename from src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityBuiltIns.cs rename to src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityConstants.cs index 347d5aa3e..2bff0e643 100644 --- a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityBuiltIns.cs +++ b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityConstants.cs @@ -6,7 +6,7 @@ namespace Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract /// /// Names of built-in session affinity services. /// - public static class SessionAffinityBuiltIns + public static class SessionAffinityConstants { public static class Modes { diff --git a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityDefaultOptions.cs b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityDefaultOptions.cs index 0f680c245..4fa7c780c 100644 --- a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityDefaultOptions.cs +++ b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityDefaultOptions.cs @@ -10,8 +10,8 @@ namespace Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract /// public class SessionAffinityDefaultOptions { - private string _defaultMode = SessionAffinityBuiltIns.Modes.Cookie; - private string _defaultAffinityFailurePolicy = SessionAffinityBuiltIns.AffinityFailurePolicies.Redistribute; + private string _defaultMode = SessionAffinityConstants.Modes.Cookie; + private string _defaultAffinityFailurePolicy = SessionAffinityConstants.AffinityFailurePolicies.Redistribute; /// /// Default session affinity mode to be used when none is specified for a backend. diff --git a/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs index 39b57c88e..521eb4f9c 100644 --- a/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs @@ -61,6 +61,7 @@ public virtual AffinityResult FindAffinitizedDestinations(HttpContext context, I { for (var i = 0; i < destinations.Count; i++) { + // TODO: Add fast destination lookup by ID if (requestAffinityKey.Key.Equals(GetDestinationAffinityKey(destinations[i]))) { // It's allowed to affinitize a request to a pool of destinations so as to enable load-balancing among them. diff --git a/src/ReverseProxy/Service/SessionAffinity/CookieSessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/CookieSessionAffinityProvider.cs index 0fa930b39..a28d225ed 100644 --- a/src/ReverseProxy/Service/SessionAffinity/CookieSessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/CookieSessionAffinityProvider.cs @@ -24,7 +24,7 @@ public CookieSessionAffinityProvider( _providerOptions = providerOptions?.Value ?? throw new ArgumentNullException(nameof(providerOptions)); } - public override string Mode => SessionAffinityBuiltIns.Modes.Cookie; + public override string Mode => SessionAffinityConstants.Modes.Cookie; protected override string GetDestinationAffinityKey(DestinationInfo destination) { diff --git a/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs index d7df833a8..bc36b3563 100644 --- a/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs @@ -20,7 +20,7 @@ public CustomHeaderSessionAffinityProvider( : base(dataProtectionProvider, logger) {} - public override string Mode => SessionAffinityBuiltIns.Modes.CustomHeader; + public override string Mode => SessionAffinityConstants.Modes.CustomHeader; protected override string GetDestinationAffinityKey(DestinationInfo destination) { diff --git a/src/ReverseProxy/Service/SessionAffinity/RedistributeAffinityFailurePolicy.cs b/src/ReverseProxy/Service/SessionAffinity/RedistributeAffinityFailurePolicy.cs index c9946fa56..f04464d7b 100644 --- a/src/ReverseProxy/Service/SessionAffinity/RedistributeAffinityFailurePolicy.cs +++ b/src/ReverseProxy/Service/SessionAffinity/RedistributeAffinityFailurePolicy.cs @@ -10,7 +10,7 @@ namespace Microsoft.ReverseProxy.Service.SessionAffinity { internal class RedistributeAffinityFailurePolicy : IAffinityFailurePolicy { - public string Name => SessionAffinityBuiltIns.AffinityFailurePolicies.Redistribute; + public string Name => SessionAffinityConstants.AffinityFailurePolicies.Redistribute; public Task Handle(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, AffinityStatus affinityStatus) { diff --git a/src/ReverseProxy/Service/SessionAffinity/Return503ErrorAffinityFailurePolicy.cs b/src/ReverseProxy/Service/SessionAffinity/Return503ErrorAffinityFailurePolicy.cs index 4b73bc02f..db752d939 100644 --- a/src/ReverseProxy/Service/SessionAffinity/Return503ErrorAffinityFailurePolicy.cs +++ b/src/ReverseProxy/Service/SessionAffinity/Return503ErrorAffinityFailurePolicy.cs @@ -10,7 +10,7 @@ namespace Microsoft.ReverseProxy.Service.SessionAffinity { internal class Return503ErrorAffinityFailurePolicy : IAffinityFailurePolicy { - public string Name => SessionAffinityBuiltIns.AffinityFailurePolicies.Return503Error; + public string Name => SessionAffinityConstants.AffinityFailurePolicies.Return503Error; public Task Handle(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, AffinityStatus affinityStatus) { diff --git a/test/ReverseProxy.Tests/Service/SessionAffinity/CookieSessionAffinityProviderTests.cs b/test/ReverseProxy.Tests/Service/SessionAffinity/CookieSessionAffinityProviderTests.cs index 6527fc74c..82ae40419 100644 --- a/test/ReverseProxy.Tests/Service/SessionAffinity/CookieSessionAffinityProviderTests.cs +++ b/test/ReverseProxy.Tests/Service/SessionAffinity/CookieSessionAffinityProviderTests.cs @@ -25,7 +25,7 @@ public void FindAffinitizedDestination_AffinityKeyIsNotSetOnRequest_ReturnKeyNot AffinityTestHelper.GetDataProtector().Object, AffinityTestHelper.GetLogger().Object); - Assert.Equal(SessionAffinityBuiltIns.Modes.Cookie, provider.Mode); + Assert.Equal(SessionAffinityConstants.Modes.Cookie, provider.Mode); var context = new DefaultHttpContext(); context.Request.Headers["Cookie"] = new[] { $"Some-Cookie=ZZZ" }; diff --git a/test/ReverseProxy.Tests/Service/SessionAffinity/CustomHeaderSessionAffinityProviderTests.cs b/test/ReverseProxy.Tests/Service/SessionAffinity/CustomHeaderSessionAffinityProviderTests.cs index d30481d50..9a4bd6275 100644 --- a/test/ReverseProxy.Tests/Service/SessionAffinity/CustomHeaderSessionAffinityProviderTests.cs +++ b/test/ReverseProxy.Tests/Service/SessionAffinity/CustomHeaderSessionAffinityProviderTests.cs @@ -22,7 +22,7 @@ public void FindAffinitizedDestination_AffinityKeyIsNotSetOnRequest_ReturnKeyNot { var provider = new CustomHeaderSessionAffinityProvider(AffinityTestHelper.GetDataProtector().Object, AffinityTestHelper.GetLogger().Object); - Assert.Equal(SessionAffinityBuiltIns.Modes.CustomHeader, provider.Mode); + Assert.Equal(SessionAffinityConstants.Modes.CustomHeader, provider.Mode); var context = new DefaultHttpContext(); context.Request.Headers["SomeHeader"] = new[] { "SomeValue" }; diff --git a/test/ReverseProxy.Tests/Service/SessionAffinity/RedistributeAffinityFailurePolicyTests.cs b/test/ReverseProxy.Tests/Service/SessionAffinity/RedistributeAffinityFailurePolicyTests.cs index 35d5eec73..3016347e7 100644 --- a/test/ReverseProxy.Tests/Service/SessionAffinity/RedistributeAffinityFailurePolicyTests.cs +++ b/test/ReverseProxy.Tests/Service/SessionAffinity/RedistributeAffinityFailurePolicyTests.cs @@ -18,7 +18,7 @@ public async Task Handle_AnyAffinitStatus_ReturnTrue(AffinityStatus status) { var policy = new RedistributeAffinityFailurePolicy(); - Assert.Equal(SessionAffinityBuiltIns.AffinityFailurePolicies.Redistribute, policy.Name); + Assert.Equal(SessionAffinityConstants.AffinityFailurePolicies.Redistribute, policy.Name); Assert.True(await policy.Handle(new DefaultHttpContext(), default, status)); } diff --git a/test/ReverseProxy.Tests/Service/SessionAffinity/Return503ErrorAffinityFailurePolicyTests.cs b/test/ReverseProxy.Tests/Service/SessionAffinity/Return503ErrorAffinityFailurePolicyTests.cs index 92121fec4..adb04f341 100644 --- a/test/ReverseProxy.Tests/Service/SessionAffinity/Return503ErrorAffinityFailurePolicyTests.cs +++ b/test/ReverseProxy.Tests/Service/SessionAffinity/Return503ErrorAffinityFailurePolicyTests.cs @@ -18,7 +18,7 @@ public async Task Handle_FaultyAffinityStatus_RespondWith503(AffinityStatus stat var policy = new Return503ErrorAffinityFailurePolicy(); var context = new DefaultHttpContext(); - Assert.Equal(SessionAffinityBuiltIns.AffinityFailurePolicies.Return503Error, policy.Name); + Assert.Equal(SessionAffinityConstants.AffinityFailurePolicies.Return503Error, policy.Name); Assert.False(await policy.Handle(context, default, status)); Assert.Equal(503, context.Response.StatusCode); From 9226fdcf295d72ffde42400711a230af234e8b7a Mon Sep 17 00:00:00 2001 From: Alexander Nikolaev Date: Wed, 3 Jun 2020 16:07:44 +0200 Subject: [PATCH 20/30] - Redundant single-item array allocations removed - AffinitizeRequest refactored - Other small fixes to address comments --- src/ReverseProxy/EventIds.cs | 7 ++-- .../Middleware/AffinitizeRequestMiddleware.cs | 35 ++++++++++--------- .../AffinitizedDestinationLookupMiddleware.cs | 4 +-- .../BaseSessionAffinityProvider.cs | 20 ++++------- .../BaseSesstionAffinityProviderTests.cs | 3 ++ 5 files changed, 33 insertions(+), 36 deletions(-) diff --git a/src/ReverseProxy/EventIds.cs b/src/ReverseProxy/EventIds.cs index 7cf75a23a..b1688cb7e 100644 --- a/src/ReverseProxy/EventIds.cs +++ b/src/ReverseProxy/EventIds.cs @@ -43,9 +43,8 @@ internal static class EventIds public static readonly EventId AffinityResolutionFailedForBackend = new EventId(34, "AffinityResolutionFailedForBackend"); public static readonly EventId MultipleDestinationsOnBackendToEstablishRequestAffinity = new EventId(35, "MultipleDestinationsOnBackendToEstablishRequestAffinity"); public static readonly EventId AffinityCannotBeEstablishedBecauseNoDestinationsFoundOnBackend = new EventId(36, "AffinityCannotBeEstablishedBecauseNoDestinationsFoundOnBackend"); - public static readonly EventId RequestAffinityToDestinationCannotBeEstablishedBecauseAffinitizationDisabled = new EventId(37, "RequestAffinityToDestinationCannotBeEstablishedBecauseAffinitizationDisabled"); - public static readonly EventId NoDestinationOnBackendToEstablishRequestAffinity = new EventId(38, "NoDestinationOnBackendToEstablishRequestAffinity"); - public static readonly EventId RequestAffinityKeyCookieCannotBeDecodedFromBase64 = new EventId(39, "RequestAffinityKeyCookieCannotBeDecodedFromBase64"); - public static readonly EventId RequestAffinityKeyCookieDecryptionFailed = new EventId(40, "RequestAffinityKeyCookieDecryptionFailed"); + public static readonly EventId NoDestinationOnBackendToEstablishRequestAffinity = new EventId(37, "NoDestinationOnBackendToEstablishRequestAffinity"); + public static readonly EventId RequestAffinityKeyCookieCannotBeDecodedFromBase64 = new EventId(38, "RequestAffinityKeyCookieCannotBeDecodedFromBase64"); + public static readonly EventId RequestAffinityKeyCookieDecryptionFailed = new EventId(39, "RequestAffinityKeyCookieDecryptionFailed"); } } diff --git a/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs b/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs index 9b0b6645c..76273d713 100644 --- a/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs +++ b/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs @@ -44,33 +44,36 @@ public Task Invoke(HttpContext context) { var destinationsFeature = context.GetRequiredDestinationFeature(); var candidateDestinations = destinationsFeature.Destinations; + if (candidateDestinations.Count == 0) { + // Only log the warning about missing destinations here, but allow the request to proceed further. + // The final check for selected destination is to be done at the pipeline end. Log.NoDestinationOnBackendToEstablishRequestAffinity(_logger, backend.BackendId); - context.Response.StatusCode = 503; - return Task.CompletedTask; } - var destinations = _operationLogger.Execute("ReverseProxy.AffinitizeRequest", () => AffinitizeRequest(context, backend, options, candidateDestinations)); - destinationsFeature.Destinations = destinations; + else + { + var chosenDestination = candidateDestinations[0]; + if (candidateDestinations.Count > 1) + { + Log.MultipleDestinationsOnBackendToEstablishRequestAffinity(_logger, backend.BackendId); + // It's assumed that all of them match to the request's affinity key. + chosenDestination = candidateDestinations[_random.Next(candidateDestinations.Count)]; + } + + _operationLogger.Execute("ReverseProxy.AffinitizeRequest", () => AffinitizeRequest(context, options, chosenDestination)); + + destinationsFeature.Destinations = chosenDestination; + } } return _next(context); } - private IReadOnlyList AffinitizeRequest(HttpContext context, BackendInfo backend, BackendConfig.BackendSessionAffinityOptions options, IReadOnlyList destinations) + private void AffinitizeRequest(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, DestinationInfo destination) { - var result = destinations; - if (result.Count > 1) - { - Log.MultipleDestinationsOnBackendToEstablishRequestAffinity(_logger, backend.BackendId); - // It's assumed that all of them match to the request's affinity key. - var singleDestination = destinations[_random.Next(destinations.Count)]; - result = new[] { singleDestination }; - } - var currentProvider = _sessionAffinityProviders.GetRequiredServiceById(options.Mode); - currentProvider.AffinitizeRequest(context, options, result[0]); - return result; + currentProvider.AffinitizeRequest(context, options, destination); } private static class Log diff --git a/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs b/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs index f03d11b3c..ec2794a89 100644 --- a/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs +++ b/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs @@ -66,13 +66,13 @@ public async Task Invoke(HttpContext context) break; case AffinityStatus.AffinityKeyExtractionFailed: case AffinityStatus.DestinationNotFound: - var failureHandled = await _operationLogger.ExecuteAsync("ReverseProxy.HandleAffinityFailure", async () => + var responseSent = await _operationLogger.ExecuteAsync("ReverseProxy.HandleAffinityFailure", async () => { var failurePolicy = _affinityFailurePolicies.GetRequiredServiceById(options.AffinityFailurePolicy); return await failurePolicy.Handle(context, options, affinityResult.Status); }); - if (!failureHandled) + if (!responseSent) { // Policy reported the failure is unrecoverable and took the full responsibility for its handling, // so we simply stop processing. diff --git a/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs index 521eb4f9c..f7df29ede 100644 --- a/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Text; using Microsoft.AspNetCore.DataProtection; using Microsoft.AspNetCore.Http; @@ -29,7 +30,7 @@ public virtual void AffinitizeRequest(HttpContext context, in BackendConfig.Back { if (!options.Enabled) { - Log.RequestAffinityToDestinationCannotBeEstablishedBecauseAffinitizationDisabled(Logger, destination.DestinationId); + Debug.Fail("AffinitizeRequest is called when session affinity is disabled"); return; } @@ -45,6 +46,7 @@ public virtual AffinityResult FindAffinitizedDestinations(HttpContext context, I { if (!options.Enabled) { + Debug.Fail("FindAffinitizedDestinations when session affinity is disabled"); // This case is handled separately to improve the type autonomy and the pipeline extensibility return new AffinityResult(null, AffinityStatus.AffinityDisabled); } @@ -56,7 +58,7 @@ public virtual AffinityResult FindAffinitizedDestinations(HttpContext context, I return new AffinityResult(null, requestAffinityKey.ExtractedSuccessfully ? AffinityStatus.AffinityKeyNotSet : AffinityStatus.AffinityKeyExtractionFailed); } - var matchingDestinations = new DestinationInfo[1]; + IReadOnlyList matchingDestinations = null; if (destinations.Count > 0) { for (var i = 0; i < destinations.Count; i++) @@ -66,7 +68,7 @@ public virtual AffinityResult FindAffinitizedDestinations(HttpContext context, I { // It's allowed to affinitize a request to a pool of destinations so as to enable load-balancing among them. // However, we currently stop after the first match found to avoid performance degradation. - matchingDestinations[0] = destinations[i]; + matchingDestinations = destinations[i]; break; } } @@ -77,7 +79,7 @@ public virtual AffinityResult FindAffinitizedDestinations(HttpContext context, I } // Empty destination list passed to this method is handled the same way as if no matching destinations are found. - if (matchingDestinations[0] == null) + if (matchingDestinations == null) { return new AffinityResult(null, AffinityStatus.DestinationNotFound); } @@ -164,11 +166,6 @@ private static class Log EventIds.AffinityCannotBeEstablishedBecauseNoDestinationsFoundOnBackend, "The request affinity cannot be established because no destinations are found on backend `{backendId}`."); - private static readonly Action _requestAffinityToDestinationCannotBeEstablishedBecauseAffinitizationDisabled = LoggerMessage.Define( - LogLevel.Warning, - EventIds.RequestAffinityToDestinationCannotBeEstablishedBecauseAffinitizationDisabled, - "The request affinity to destination `{destinationId}` cannot be established because affinitization is disabled for the backend."); - private static readonly Action _requestAffinityKeyCookieDecryptionFailed = LoggerMessage.Define( LogLevel.Error, EventIds.RequestAffinityKeyCookieDecryptionFailed, @@ -184,11 +181,6 @@ public static void AffinityCannotBeEstablishedBecauseNoDestinationsFound(ILogger _affinityCannotBeEstablishedBecauseNoDestinationsFound(logger, backendId, null); } - public static void RequestAffinityToDestinationCannotBeEstablishedBecauseAffinitizationDisabled(ILogger logger, string destinationId) - { - _requestAffinityToDestinationCannotBeEstablishedBecauseAffinitizationDisabled(logger, destinationId, null); - } - public static void RequestAffinityKeyCookieDecryptionFailed(ILogger logger, Exception ex) { _requestAffinityKeyCookieDecryptionFailed(logger, ex); diff --git a/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTests.cs b/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTests.cs index d93566d8f..f9c425c44 100644 --- a/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTests.cs +++ b/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTests.cs @@ -53,6 +53,8 @@ public void Request_FindAffinitizedDestinations( } } + // Current version of test SDK cannot properly handle Debug.Fail, so the tests are skipped in Debug +#if RELEASE [Fact] public void FindAffinitizedDestination_AffinityDisabledOnBackend_ReturnsAffinityDisabled() { @@ -72,6 +74,7 @@ public void AffinitizeRequest_AffinitiDisabled_DoNothing() Assert.Null(provider.LastSetEncryptedKey); dataProtector.Verify(p => p.Protect(It.IsAny()), Times.Never); } +#endif [Fact] public void AffinitizeRequest_RequestIsAffinitized_DoNothing() From d375c2852c82ed3b075f33ef1d159b3a527f2abd Mon Sep 17 00:00:00 2001 From: Alexander Nikolaev Date: Wed, 3 Jun 2020 17:07:40 +0200 Subject: [PATCH 21/30] DestinationInfo comparison fixed --- .../Middleware/AffinitizedDestinationLookupMiddlewareTests.cs | 2 +- .../SessionAffinity/BaseSesstionAffinityProviderTests.cs | 2 +- .../SessionAffinity/CookieSessionAffinityProviderTests.cs | 2 +- .../SessionAffinity/CustomHeaderSessionAffinityProviderTests.cs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/ReverseProxy.Tests/Middleware/AffinitizedDestinationLookupMiddlewareTests.cs b/test/ReverseProxy.Tests/Middleware/AffinitizedDestinationLookupMiddlewareTests.cs index 67fc07dd0..cc000c79f 100644 --- a/test/ReverseProxy.Tests/Middleware/AffinitizedDestinationLookupMiddlewareTests.cs +++ b/test/ReverseProxy.Tests/Middleware/AffinitizedDestinationLookupMiddlewareTests.cs @@ -61,7 +61,7 @@ public async Task Invoke_SuccessfulFlow_CallNext(AffinityStatus status, string f } else { - Assert.Equal(_destinations, destinationFeature.Destinations); + Assert.Same(_destinations, destinationFeature.Destinations); } } diff --git a/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTests.cs b/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTests.cs index f9c425c44..775d00f57 100644 --- a/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTests.cs +++ b/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTests.cs @@ -42,7 +42,7 @@ public void Request_FindAffinitizedDestinations( } Assert.Equal(expectedStatus, affinityResult.Status); - Assert.Equal(expectedDestination, affinityResult.Destinations?.FirstOrDefault()); + Assert.Same(expectedDestination, affinityResult.Destinations?.FirstOrDefault()); if (expectedDestination != null) { Assert.Equal(1, affinityResult.Destinations.Count); diff --git a/test/ReverseProxy.Tests/Service/SessionAffinity/CookieSessionAffinityProviderTests.cs b/test/ReverseProxy.Tests/Service/SessionAffinity/CookieSessionAffinityProviderTests.cs index 82ae40419..d4e24f2e8 100644 --- a/test/ReverseProxy.Tests/Service/SessionAffinity/CookieSessionAffinityProviderTests.cs +++ b/test/ReverseProxy.Tests/Service/SessionAffinity/CookieSessionAffinityProviderTests.cs @@ -51,7 +51,7 @@ public void FindAffinitizedDestination_AffinityKeyIsSetOnRequest_Success() Assert.Equal(AffinityStatus.OK, affinityResult.Status); Assert.Equal(1, affinityResult.Destinations.Count); - Assert.Equal(affinitizedDestination, affinityResult.Destinations[0]); + Assert.Same(affinitizedDestination, affinityResult.Destinations[0]); } [Fact] diff --git a/test/ReverseProxy.Tests/Service/SessionAffinity/CustomHeaderSessionAffinityProviderTests.cs b/test/ReverseProxy.Tests/Service/SessionAffinity/CustomHeaderSessionAffinityProviderTests.cs index 9a4bd6275..00bc63ec1 100644 --- a/test/ReverseProxy.Tests/Service/SessionAffinity/CustomHeaderSessionAffinityProviderTests.cs +++ b/test/ReverseProxy.Tests/Service/SessionAffinity/CustomHeaderSessionAffinityProviderTests.cs @@ -46,7 +46,7 @@ public void FindAffinitizedDestination_AffinityKeyIsSetOnRequest_Success() Assert.Equal(AffinityStatus.OK, affinityResult.Status); Assert.Equal(1, affinityResult.Destinations.Count); - Assert.Equal(affinitizedDestination, affinityResult.Destinations[0]); + Assert.Same(affinitizedDestination, affinityResult.Destinations[0]); } [Fact] From dbf9fffb1ff31ab9af78bec34382eefece9ef135 Mon Sep 17 00:00:00 2001 From: Alexander Nikolaev Date: Thu, 4 Jun 2020 11:51:42 +0200 Subject: [PATCH 22/30] - AffinitizeRequestMiddleware tests - Naming fix --- .../Middleware/AffinitizeRequestMiddleware.cs | 2 +- .../AffinitizedDestinationLookupMiddleware.cs | 4 +- .../SessionAffinityMiddlewareHelper.cs | 4 +- .../AffinitizeRequestMiddlewareTests.cs | 140 ++++++++++++++++++ ...nitizedDestinationLookupMiddlewareTests.cs | 95 +++--------- .../Middleware/AffinityMiddlewareTestBase.cs | 87 +++++++++++ 6 files changed, 253 insertions(+), 79 deletions(-) create mode 100644 test/ReverseProxy.Tests/Middleware/AffinitizeRequestMiddlewareTests.cs create mode 100644 test/ReverseProxy.Tests/Middleware/AffinityMiddlewareTestBase.cs diff --git a/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs b/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs index 76273d713..3f3b7e86a 100644 --- a/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs +++ b/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs @@ -84,7 +84,7 @@ private static class Log "The request still has multiple destinations on the backend `{backendId}` to choose from when establishing affinity, load balancing may not be properly configured. A random destination will be used."); private static readonly Action _noDestinationOnBackendToEstablishRequestAffinity = LoggerMessage.Define( - LogLevel.Error, + LogLevel.Warning, EventIds.NoDestinationOnBackendToEstablishRequestAffinity, "The request doesn't have any destinations on the backend `{backendId}` to choose from when establishing affinity, load balancing may not be properly configured."); diff --git a/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs b/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs index ec2794a89..17e72687d 100644 --- a/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs +++ b/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs @@ -66,13 +66,13 @@ public async Task Invoke(HttpContext context) break; case AffinityStatus.AffinityKeyExtractionFailed: case AffinityStatus.DestinationNotFound: - var responseSent = await _operationLogger.ExecuteAsync("ReverseProxy.HandleAffinityFailure", async () => + var keepProcessing = await _operationLogger.ExecuteAsync("ReverseProxy.HandleAffinityFailure", async () => { var failurePolicy = _affinityFailurePolicies.GetRequiredServiceById(options.AffinityFailurePolicy); return await failurePolicy.Handle(context, options, affinityResult.Status); }); - if (!responseSent) + if (!keepProcessing) { // Policy reported the failure is unrecoverable and took the full responsibility for its handling, // so we simply stop processing. diff --git a/src/ReverseProxy/Service/SessionAffinity/SessionAffinityMiddlewareHelper.cs b/src/ReverseProxy/Service/SessionAffinity/SessionAffinityMiddlewareHelper.cs index 8152d53d1..f99cde986 100644 --- a/src/ReverseProxy/Service/SessionAffinity/SessionAffinityMiddlewareHelper.cs +++ b/src/ReverseProxy/Service/SessionAffinity/SessionAffinityMiddlewareHelper.cs @@ -21,7 +21,7 @@ public static IDictionary ToDictionaryById(this IEnumerable ser { if (!result.TryAdd(idSelector(service), service)) { - throw new ArgumentException(nameof(services), $"More than one {nameof(T)} found with the same identifier."); + throw new ArgumentException(nameof(services), $"More than one {typeof(T)} found with the same identifier."); } } @@ -42,7 +42,7 @@ public static T GetRequiredServiceById(this IDictionary services, { if (!services.TryGetValue(id, out var result)) { - throw new ArgumentException(nameof(id), $"No {nameof(T)} was found for the id {id}."); + throw new ArgumentException(nameof(id), $"No {typeof(T)} was found for the id {id}."); } return result; } diff --git a/test/ReverseProxy.Tests/Middleware/AffinitizeRequestMiddlewareTests.cs b/test/ReverseProxy.Tests/Middleware/AffinitizeRequestMiddlewareTests.cs new file mode 100644 index 000000000..15bfd3a59 --- /dev/null +++ b/test/ReverseProxy.Tests/Middleware/AffinitizeRequestMiddlewareTests.cs @@ -0,0 +1,140 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using Microsoft.ReverseProxy.Abstractions.Telemetry; +using Microsoft.ReverseProxy.RuntimeModel; +using Microsoft.ReverseProxy.Service.SessionAffinity; +using Moq; +using Xunit; + +namespace Microsoft.ReverseProxy.Middleware +{ + public class AffinitizeRequestMiddlewareTests : AffinityMiddlewareTestBase + { + [Fact] + public async Task Invoke_SingleDestinationChosen_InvokeAffinitizeRequest() + { + var backend = GetBackend(); + var invokedMode = string.Empty; + const string expectedMode = "Mode-B"; + var providers = RegisterAffinityProviders( + false, + Destinations[1], + backend.BackendId, + ("Mode-A", (AffinityStatus?)null, (DestinationInfo[])null, (Action)(p => throw new InvalidOperationException($"Provider {p.Mode} call is not expected."))), + (expectedMode, (AffinityStatus?)null, (DestinationInfo[])null, (Action)(p => invokedMode = p.Mode))); + var nextInvoked = false; + var middleware = new AffinitizeRequestMiddleware(c => { + nextInvoked = true; + return Task.CompletedTask; + }, + providers.Select(p => p.Object), + GetOperationLogger(), + new Mock>().Object); + var context = new DefaultHttpContext(); + context.Features.Set(backend); + var destinationFeature = GetDestinationsFeature(Destinations[1]); + context.Features.Set(destinationFeature); + + await middleware.Invoke(context); + + Assert.Equal(expectedMode, invokedMode); + Assert.True(nextInvoked); + providers[0].VerifyGet(p => p.Mode, Times.Once); + providers[0].VerifyNoOtherCalls(); + providers[1].VerifyAll(); + Assert.Same(destinationFeature.Destinations, Destinations[1]); + } + + [Fact] + public async Task Invoke_MultipleCandidateDestinations_ChooseOneAndInvokeAffinitizeRequest() + { + var backend = GetBackend(); + var invokedMode = string.Empty; + const string expectedMode = "Mode-B"; + var providers = new[] { + GetProviderForRandomDestination("Mode-A", Destinations, p => throw new InvalidOperationException($"Provider {p.Mode} call is not expected.")), + GetProviderForRandomDestination(expectedMode, Destinations, p => invokedMode = p.Mode) + }; + var nextInvoked = false; + var logger = new Mock>(); + logger.Setup(l => l.IsEnabled(LogLevel.Warning)).Returns(true); + var middleware = new AffinitizeRequestMiddleware(c => { + nextInvoked = true; + return Task.CompletedTask; + }, + providers.Select(p => p.Object), + GetOperationLogger(), + logger.Object); + var context = new DefaultHttpContext(); + context.Features.Set(backend); + var destinationFeature = GetDestinationsFeature(Destinations); + context.Features.Set(destinationFeature); + + await middleware.Invoke(context); + + Assert.Equal(expectedMode, invokedMode); + Assert.True(nextInvoked); + providers[0].VerifyGet(p => p.Mode, Times.Once); + providers[0].VerifyNoOtherCalls(); + providers[1].VerifyAll(); + logger.Verify( + l => l.Log(LogLevel.Warning, EventIds.MultipleDestinationsOnBackendToEstablishRequestAffinity, It.IsAny(), null, (Func)It.IsAny()), + Times.Once); + Assert.Equal(1, destinationFeature.Destinations.Count); + var chosen = destinationFeature.Destinations[0]; + var sameDestinationCount = Destinations.Count(d => chosen == d); + Assert.Equal(1, sameDestinationCount); + } + + [Fact] + public async Task Invoke_NoDestinationChosen_LogWarningAndCallNext() + { + var backend = GetBackend(); + var nextInvoked = false; + var logger = new Mock>(); + logger.Setup(l => l.IsEnabled(LogLevel.Warning)).Returns(true); + var middleware = new AffinitizeRequestMiddleware(c => { + nextInvoked = true; + return Task.CompletedTask; + }, + new ISessionAffinityProvider[0], + GetOperationLogger(), + logger.Object); + var context = new DefaultHttpContext(); + context.Features.Set(backend); + var destinationFeature = GetDestinationsFeature(new DestinationInfo[0]); + context.Features.Set(destinationFeature); + + await middleware.Invoke(context); + + Assert.True(nextInvoked); + logger.Verify( + l => l.Log(LogLevel.Warning, EventIds.NoDestinationOnBackendToEstablishRequestAffinity, It.IsAny(), null, (Func)It.IsAny()), + Times.Once); + Assert.Equal(0, destinationFeature.Destinations.Count); + } + + private IOperationLogger GetOperationLogger() + { + var result = new Mock>(MockBehavior.Strict); + result.Setup(l => l.Execute(It.IsAny(), It.IsAny())).Callback((string name, Action callback) => callback()); + return result.Object; + } + + private Mock GetProviderForRandomDestination(string mode, IReadOnlyList destinations, Action callback) + { + var provider = new Mock(MockBehavior.Strict); + provider.SetupGet(p => p.Mode).Returns(mode); + provider.Setup(p => p.AffinitizeRequest(It.IsAny(), BackendConfig.SessionAffinityOptions, It.Is(d => destinations.Contains(d)))) + .Callback(() => callback(provider.Object)); + return provider; + } + } +} diff --git a/test/ReverseProxy.Tests/Middleware/AffinitizedDestinationLookupMiddlewareTests.cs b/test/ReverseProxy.Tests/Middleware/AffinitizedDestinationLookupMiddlewareTests.cs index cc000c79f..bf4d40e95 100644 --- a/test/ReverseProxy.Tests/Middleware/AffinitizedDestinationLookupMiddlewareTests.cs +++ b/test/ReverseProxy.Tests/Middleware/AffinitizedDestinationLookupMiddlewareTests.cs @@ -2,15 +2,11 @@ // Licensed under the MIT License. using System; -using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; using Microsoft.ReverseProxy.Abstractions.Telemetry; -using Microsoft.ReverseProxy.RuntimeModel; -using Microsoft.ReverseProxy.Service.Management; -using Microsoft.ReverseProxy.Service.Proxy.Infrastructure; using Microsoft.ReverseProxy.Service.SessionAffinity; using Microsoft.ReverseProxy.Signals; using Moq; @@ -18,12 +14,8 @@ namespace Microsoft.ReverseProxy.Middleware { - public class AffinitizedDestinationLookupMiddlewareTests + public class AffinitizedDestinationLookupMiddlewareTests : AffinityMiddlewareTestBase { - private const string AffinitizedDestinationName = "dest-B"; - private readonly IReadOnlyList _destinations = new[] { new DestinationInfo("dest-A"), new DestinationInfo(AffinitizedDestinationName), new DestinationInfo("dest-C") }; - private readonly BackendConfig _backendConfig = new BackendConfig(default, default, new BackendConfig.BackendSessionAffinityOptions(true, "Mode-B", "Policy-1", null)); - [Theory] [InlineData(AffinityStatus.AffinityKeyNotSet, null)] [InlineData(AffinityStatus.AffinityDisabled, null)] @@ -31,25 +23,32 @@ public class AffinitizedDestinationLookupMiddlewareTests public async Task Invoke_SuccessfulFlow_CallNext(AffinityStatus status, string foundDestinationId) { var backend = GetBackend(); - var foundDestinations = foundDestinationId != null ? _destinations.Where(d => d.DestinationId == foundDestinationId).ToArray() : null; + var foundDestinations = foundDestinationId != null ? Destinations.Where(d => d.DestinationId == foundDestinationId).ToArray() : null; var invokedMode = string.Empty; + const string expectedMode = "Mode-B"; var providers = RegisterAffinityProviders( - _destinations, + true, + Destinations, backend.BackendId, - ("Mode-A", AffinityStatus.DestinationNotFound, null, p => throw new InvalidOperationException($"Provider {p.Mode} call is not expected.")), - ("Mode-B", status, foundDestinations, p => invokedMode = p.Mode)); - var middleware = new AffinitizedDestinationLookupMiddleware(c => Task.CompletedTask, + ("Mode-A", AffinityStatus.DestinationNotFound, (RuntimeModel.DestinationInfo[])null, (Action)(p => throw new InvalidOperationException($"Provider {p.Mode} call is not expected."))), + (expectedMode, status, foundDestinations, p => invokedMode = p.Mode)); + var nextInvoked = false; + var middleware = new AffinitizedDestinationLookupMiddleware(c => { + nextInvoked = true; + return Task.CompletedTask; + }, providers.Select(p => p.Object), new IAffinityFailurePolicy[0], GetOperationLogger(false), new Mock>().Object); var context = new DefaultHttpContext(); context.Features.Set(backend); - var destinationFeature = GetDestinationsFeature(_destinations); + var destinationFeature = GetDestinationsFeature(Destinations); context.Features.Set(destinationFeature); await middleware.Invoke(context); - Assert.Equal("Mode-B", invokedMode); + Assert.Equal(expectedMode, invokedMode); + Assert.True(nextInvoked); providers[0].VerifyGet(p => p.Mode, Times.Once); providers[0].VerifyNoOtherCalls(); providers[1].VerifyAll(); @@ -61,7 +60,7 @@ public async Task Invoke_SuccessfulFlow_CallNext(AffinityStatus status, string f } else { - Assert.Same(_destinations, destinationFeature.Destinations); + Assert.Same(Destinations, destinationFeature.Destinations); } } @@ -73,12 +72,13 @@ public async Task Invoke_SuccessfulFlow_CallNext(AffinityStatus status, string f public async Task Invoke_ErrorFlow_CallFailurePolicy(AffinityStatus affinityStatus, bool handled) { var backend = GetBackend(); - var providers = RegisterAffinityProviders(_destinations, backend.BackendId, ("Mode-B", affinityStatus, null, _ => { })); + var providers = RegisterAffinityProviders(true, Destinations, backend.BackendId, ("Mode-B", affinityStatus, null, _ => { })); var invokedPolicy = string.Empty; + const string expectedPolicy = "Policy-1"; var failurePolicies = RegisterFailurePolicies( affinityStatus, ("Policy-0", false, p => throw new InvalidOperationException($"Policy {p.Name} call is not expected.")), - ("Policy-1", handled, p => invokedPolicy = p.Name)); + (expectedPolicy, handled, p => invokedPolicy = p.Name)); var nextInvoked = false; var middleware = new AffinitizedDestinationLookupMiddleware(c => { nextInvoked = true; @@ -89,71 +89,18 @@ public async Task Invoke_ErrorFlow_CallFailurePolicy(AffinityStatus affinityStat new Mock>().Object); var context = new DefaultHttpContext(); context.Features.Set(backend); - var destinationFeature = GetDestinationsFeature(_destinations); + var destinationFeature = GetDestinationsFeature(Destinations); context.Features.Set(destinationFeature); await middleware.Invoke(context); - Assert.Equal("Policy-1", invokedPolicy); + Assert.Equal(expectedPolicy, invokedPolicy); Assert.Equal(handled, nextInvoked); failurePolicies[0].VerifyGet(p => p.Name, Times.Once); failurePolicies[0].VerifyNoOtherCalls(); failurePolicies[1].VerifyAll(); } - private BackendInfo GetBackend() - { - var destinationManager = new Mock(); - destinationManager.SetupGet(m => m.Items).Returns(SignalFactory.Default.CreateSignal(_destinations)); - var backend = new BackendInfo("backend-1", destinationManager.Object, new Mock().Object); - backend.Config.Value = _backendConfig; - return backend; - } - - private IReadOnlyList> RegisterAffinityProviders( - IReadOnlyList expectedDestinations, - string expectedBackend, - params (string Mode, AffinityStatus Status, DestinationInfo[] Destinations, Action Callback)[] prototypes) - { - var result = new List>(); - foreach (var (mode, status, destinations, callback) in prototypes) - { - var provider = new Mock(MockBehavior.Strict); - provider.SetupGet(p => p.Mode).Returns(mode); - provider.Setup(p => p.FindAffinitizedDestinations( - It.IsAny(), - expectedDestinations, - expectedBackend, - _backendConfig.SessionAffinityOptions)) - .Returns(new AffinityResult(destinations, status)) - .Callback(() => callback(provider.Object)); - result.Add(provider); - } - return result.AsReadOnly(); - } - - private IReadOnlyList> RegisterFailurePolicies(AffinityStatus expectedStatus, params (string Name, bool Handled, Action Callback)[] prototypes) - { - var result = new List>(); - foreach (var (name, handled, callback) in prototypes) - { - var policy = new Mock(MockBehavior.Strict); - policy.SetupGet(p => p.Name).Returns(name); - policy.Setup(p => p.Handle(It.IsAny(), It.Is(o => o.AffinityFailurePolicy == name), expectedStatus)) - .ReturnsAsync(handled) - .Callback(() => callback(policy.Object)); - result.Add(policy); - } - return result.AsReadOnly(); - } - - private IAvailableDestinationsFeature GetDestinationsFeature(IReadOnlyList destinations) - { - var result = new Mock(MockBehavior.Strict); - result.SetupProperty(p => p.Destinations, destinations); - return result.Object; - } - private IOperationLogger GetOperationLogger(bool callFailurePolicy) { var result = new Mock>(MockBehavior.Strict); diff --git a/test/ReverseProxy.Tests/Middleware/AffinityMiddlewareTestBase.cs b/test/ReverseProxy.Tests/Middleware/AffinityMiddlewareTestBase.cs new file mode 100644 index 000000000..3ee3c9673 --- /dev/null +++ b/test/ReverseProxy.Tests/Middleware/AffinityMiddlewareTestBase.cs @@ -0,0 +1,87 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using Microsoft.AspNetCore.Http; +using Microsoft.ReverseProxy.RuntimeModel; +using Microsoft.ReverseProxy.Service.Management; +using Microsoft.ReverseProxy.Service.Proxy.Infrastructure; +using Microsoft.ReverseProxy.Service.SessionAffinity; +using Microsoft.ReverseProxy.Signals; +using Moq; + +namespace Microsoft.ReverseProxy.Middleware +{ + public abstract class AffinityMiddlewareTestBase + { + protected const string AffinitizedDestinationName = "dest-B"; + protected readonly IReadOnlyList Destinations = new[] { new DestinationInfo("dest-A"), new DestinationInfo(AffinitizedDestinationName), new DestinationInfo("dest-C") }; + protected readonly BackendConfig BackendConfig = new BackendConfig(default, default, new BackendConfig.BackendSessionAffinityOptions(true, "Mode-B", "Policy-1", null)); + + internal BackendInfo GetBackend() + { + var destinationManager = new Mock(); + destinationManager.SetupGet(m => m.Items).Returns(SignalFactory.Default.CreateSignal(Destinations)); + var backend = new BackendInfo("backend-1", destinationManager.Object, new Mock().Object); + backend.Config.Value = BackendConfig; + return backend; + } + + internal IReadOnlyList> RegisterAffinityProviders( + bool lookupMiddlewareTest, + IReadOnlyList expectedDestinations, + string expectedBackend, + params (string Mode, AffinityStatus? Status, DestinationInfo[] Destinations, Action Callback)[] prototypes) + { + var result = new List>(); + foreach (var (mode, status, destinations, callback) in prototypes) + { + var provider = new Mock(MockBehavior.Strict); + provider.SetupGet(p => p.Mode).Returns(mode); + if (lookupMiddlewareTest) + { + provider.Setup(p => p.FindAffinitizedDestinations( + It.IsAny(), + expectedDestinations, + expectedBackend, + BackendConfig.SessionAffinityOptions)) + .Returns(new AffinityResult(destinations, status.Value)) + .Callback(() => callback(provider.Object)); + } + else + { + provider.Setup(p => p.AffinitizeRequest( + It.IsAny(), + BackendConfig.SessionAffinityOptions, + expectedDestinations[0])) + .Callback(() => callback(provider.Object)); + } + result.Add(provider); + } + return result.AsReadOnly(); + } + + internal IReadOnlyList> RegisterFailurePolicies(AffinityStatus expectedStatus, params (string Name, bool Handled, Action Callback)[] prototypes) + { + var result = new List>(); + foreach (var (name, handled, callback) in prototypes) + { + var policy = new Mock(MockBehavior.Strict); + policy.SetupGet(p => p.Name).Returns(name); + policy.Setup(p => p.Handle(It.IsAny(), It.Is(o => o.AffinityFailurePolicy == name), expectedStatus)) + .ReturnsAsync(handled) + .Callback(() => callback(policy.Object)); + result.Add(policy); + } + return result.AsReadOnly(); + } + + internal IAvailableDestinationsFeature GetDestinationsFeature(IReadOnlyList destinations) + { + var result = new Mock(MockBehavior.Strict); + result.SetupProperty(p => p.Destinations, destinations); + return result.Object; + } + } +} From 81de6c166ba14dfd17e6134db9a2f9a26a4be6a3 Mon Sep 17 00:00:00 2001 From: Alexander Nikolaev Date: Thu, 4 Jun 2020 11:58:01 +0200 Subject: [PATCH 23/30] - BaseSessionAffinityProvider throws exception if affinity is disabled - AffinityDisabled status is removed --- .../AffinitizedDestinationLookupMiddleware.cs | 2 -- .../Service/SessionAffinity/AffinityStatus.cs | 1 - .../SessionAffinity/BaseSessionAffinityProvider.cs | 7 ++----- .../Return503ErrorAffinityFailurePolicy.cs | 3 +-- .../AffinitizedDestinationLookupMiddlewareTests.cs | 1 - .../BaseSesstionAffinityProviderTests.cs | 11 ++--------- .../Return503ErrorAffinityFailurePolicyTests.cs | 1 - 7 files changed, 5 insertions(+), 21 deletions(-) diff --git a/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs b/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs index 17e72687d..812b3de08 100644 --- a/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs +++ b/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs @@ -59,8 +59,6 @@ public async Task Invoke(HttpContext context) case AffinityStatus.OK: destinationsFeature.Destinations = affinityResult.Destinations; break; - // This implementation treat them the same. - case AffinityStatus.AffinityDisabled: case AffinityStatus.AffinityKeyNotSet: // Nothing to do so just continue processing break; diff --git a/src/ReverseProxy/Service/SessionAffinity/AffinityStatus.cs b/src/ReverseProxy/Service/SessionAffinity/AffinityStatus.cs index d60e8d473..b2d2a0529 100644 --- a/src/ReverseProxy/Service/SessionAffinity/AffinityStatus.cs +++ b/src/ReverseProxy/Service/SessionAffinity/AffinityStatus.cs @@ -9,7 +9,6 @@ namespace Microsoft.ReverseProxy.Service.SessionAffinity public enum AffinityStatus { OK, - AffinityDisabled, AffinityKeyNotSet, AffinityKeyExtractionFailed, DestinationNotFound diff --git a/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs index f7df29ede..dcb3c0ab0 100644 --- a/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs @@ -30,8 +30,7 @@ public virtual void AffinitizeRequest(HttpContext context, in BackendConfig.Back { if (!options.Enabled) { - Debug.Fail("AffinitizeRequest is called when session affinity is disabled"); - return; + throw new InvalidOperationException($"Session affinity is disabled for backend."); } // Affinity key is set on the response only if it's a new affinity. @@ -46,9 +45,7 @@ public virtual AffinityResult FindAffinitizedDestinations(HttpContext context, I { if (!options.Enabled) { - Debug.Fail("FindAffinitizedDestinations when session affinity is disabled"); - // This case is handled separately to improve the type autonomy and the pipeline extensibility - return new AffinityResult(null, AffinityStatus.AffinityDisabled); + throw new InvalidOperationException($"Session affinity is disabled for backend {backendId}."); } var requestAffinityKey = GetRequestAffinityKey(context, options); diff --git a/src/ReverseProxy/Service/SessionAffinity/Return503ErrorAffinityFailurePolicy.cs b/src/ReverseProxy/Service/SessionAffinity/Return503ErrorAffinityFailurePolicy.cs index db752d939..a8a001c69 100644 --- a/src/ReverseProxy/Service/SessionAffinity/Return503ErrorAffinityFailurePolicy.cs +++ b/src/ReverseProxy/Service/SessionAffinity/Return503ErrorAffinityFailurePolicy.cs @@ -15,8 +15,7 @@ internal class Return503ErrorAffinityFailurePolicy : IAffinityFailurePolicy public Task Handle(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, AffinityStatus affinityStatus) { if (affinityStatus == AffinityStatus.OK - || affinityStatus == AffinityStatus.AffinityKeyNotSet - || affinityStatus == AffinityStatus.AffinityDisabled) + || affinityStatus == AffinityStatus.AffinityKeyNotSet) { // We shouldn't get here, but allow the request to proceed further if that's the case. return Task.FromResult(true); diff --git a/test/ReverseProxy.Tests/Middleware/AffinitizedDestinationLookupMiddlewareTests.cs b/test/ReverseProxy.Tests/Middleware/AffinitizedDestinationLookupMiddlewareTests.cs index bf4d40e95..b93d1c4fe 100644 --- a/test/ReverseProxy.Tests/Middleware/AffinitizedDestinationLookupMiddlewareTests.cs +++ b/test/ReverseProxy.Tests/Middleware/AffinitizedDestinationLookupMiddlewareTests.cs @@ -18,7 +18,6 @@ public class AffinitizedDestinationLookupMiddlewareTests : AffinityMiddlewareTes { [Theory] [InlineData(AffinityStatus.AffinityKeyNotSet, null)] - [InlineData(AffinityStatus.AffinityDisabled, null)] [InlineData(AffinityStatus.OK, AffinitizedDestinationName)] public async Task Invoke_SuccessfulFlow_CallNext(AffinityStatus status, string foundDestinationId) { diff --git a/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTests.cs b/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTests.cs index 775d00f57..306773244 100644 --- a/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTests.cs +++ b/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTests.cs @@ -53,16 +53,12 @@ public void Request_FindAffinitizedDestinations( } } - // Current version of test SDK cannot properly handle Debug.Fail, so the tests are skipped in Debug -#if RELEASE [Fact] public void FindAffinitizedDestination_AffinityDisabledOnBackend_ReturnsAffinityDisabled() { var provider = new ProviderStub(GetDataProtector().Object, Mock().Object); var options = new BackendConfig.BackendSessionAffinityOptions(false, _defaultOptions.Mode, _defaultOptions.AffinityFailurePolicy, _defaultOptions.Settings); - var affinityResult = provider.FindAffinitizedDestinations(new DefaultHttpContext(), new[] { new DestinationInfo("1") }, "backend-1", options); - Assert.Equal(AffinityStatus.AffinityDisabled, affinityResult.Status); - Assert.Null(affinityResult.Destinations); + Assert.Throws(() => provider.FindAffinitizedDestinations(new DefaultHttpContext(), new[] { new DestinationInfo("1") }, "backend-1", options)); } [Fact] @@ -70,11 +66,8 @@ public void AffinitizeRequest_AffinitiDisabled_DoNothing() { var dataProtector = GetDataProtector(); var provider = new ProviderStub(dataProtector.Object, Mock().Object); - provider.AffinitizeRequest(new DefaultHttpContext(), default, new DestinationInfo("id")); - Assert.Null(provider.LastSetEncryptedKey); - dataProtector.Verify(p => p.Protect(It.IsAny()), Times.Never); + Assert.Throws(() => provider.AffinitizeRequest(new DefaultHttpContext(), default, new DestinationInfo("id"))); } -#endif [Fact] public void AffinitizeRequest_RequestIsAffinitized_DoNothing() diff --git a/test/ReverseProxy.Tests/Service/SessionAffinity/Return503ErrorAffinityFailurePolicyTests.cs b/test/ReverseProxy.Tests/Service/SessionAffinity/Return503ErrorAffinityFailurePolicyTests.cs index adb04f341..f6335ae36 100644 --- a/test/ReverseProxy.Tests/Service/SessionAffinity/Return503ErrorAffinityFailurePolicyTests.cs +++ b/test/ReverseProxy.Tests/Service/SessionAffinity/Return503ErrorAffinityFailurePolicyTests.cs @@ -27,7 +27,6 @@ public async Task Handle_FaultyAffinityStatus_RespondWith503(AffinityStatus stat [Theory] [InlineData(AffinityStatus.OK)] [InlineData(AffinityStatus.AffinityKeyNotSet)] - [InlineData(AffinityStatus.AffinityDisabled)] public async Task Handle_SuccessfulAffinityStatus_ReturnTrue(AffinityStatus status) { var policy = new Return503ErrorAffinityFailurePolicy(); From f1f545f60dafdd4c01db6bab2643189e140bf513 Mon Sep 17 00:00:00 2001 From: Alexander Nikolaev Date: Thu, 4 Jun 2020 13:54:59 +0200 Subject: [PATCH 24/30] Logging covered by tests --- src/ReverseProxy/EventIds.cs | 4 +- .../BaseSessionAffinityProvider.cs | 32 +++++------ .../AffinitizeRequestMiddlewareTests.cs | 6 +-- ...nitizedDestinationLookupMiddlewareTests.cs | 15 ++++-- .../SessionAffinity/AffinityTestHelper.cs | 4 +- .../BaseSesstionAffinityProviderTests.cs | 53 +++++++++++-------- 6 files changed, 64 insertions(+), 50 deletions(-) diff --git a/src/ReverseProxy/EventIds.cs b/src/ReverseProxy/EventIds.cs index b1688cb7e..502f8f732 100644 --- a/src/ReverseProxy/EventIds.cs +++ b/src/ReverseProxy/EventIds.cs @@ -44,7 +44,7 @@ internal static class EventIds public static readonly EventId MultipleDestinationsOnBackendToEstablishRequestAffinity = new EventId(35, "MultipleDestinationsOnBackendToEstablishRequestAffinity"); public static readonly EventId AffinityCannotBeEstablishedBecauseNoDestinationsFoundOnBackend = new EventId(36, "AffinityCannotBeEstablishedBecauseNoDestinationsFoundOnBackend"); public static readonly EventId NoDestinationOnBackendToEstablishRequestAffinity = new EventId(37, "NoDestinationOnBackendToEstablishRequestAffinity"); - public static readonly EventId RequestAffinityKeyCookieCannotBeDecodedFromBase64 = new EventId(38, "RequestAffinityKeyCookieCannotBeDecodedFromBase64"); - public static readonly EventId RequestAffinityKeyCookieDecryptionFailed = new EventId(39, "RequestAffinityKeyCookieDecryptionFailed"); + public static readonly EventId RequestAffinityKeyDecryptionFailed = new EventId(38, "RequestAffinityKeyDecryptionFailed"); + public static readonly EventId DestinationMatchingToAffinityKeyNotFound = new EventId(39, "DestinationMatchingToAffinityKeyNotFound"); } } diff --git a/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs index dcb3c0ab0..8c52fca92 100644 --- a/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs @@ -69,6 +69,7 @@ public virtual AffinityResult FindAffinitizedDestinations(HttpContext context, I break; } } + Log.DestinationMatchingToAffinityKeyNotFound(Logger, backendId); } else { @@ -124,16 +125,11 @@ protected string Protect(string unencryptedKey) try { var keyBytes = Convert.FromBase64String(Pad(encryptedRequestKey)); - if (keyBytes == null) - { - Log.RequestAffinityKeyCookieCannotBeDecodedFromBase64(Logger); - return (Key: null, ExtractedSuccessfully: false); - } var decryptedKeyBytes = _dataProtector.Unprotect(keyBytes); if (decryptedKeyBytes == null) { - Log.RequestAffinityKeyCookieDecryptionFailed(Logger, null); + Log.RequestAffinityKeyDecryptionFailed(Logger, null); return (Key: null, ExtractedSuccessfully: false); } @@ -141,7 +137,7 @@ protected string Protect(string unencryptedKey) } catch (Exception ex) { - Log.RequestAffinityKeyCookieDecryptionFailed(Logger, ex); + Log.RequestAffinityKeyDecryptionFailed(Logger, ex); return (Key: null, ExtractedSuccessfully: false); } } @@ -163,29 +159,29 @@ private static class Log EventIds.AffinityCannotBeEstablishedBecauseNoDestinationsFoundOnBackend, "The request affinity cannot be established because no destinations are found on backend `{backendId}`."); - private static readonly Action _requestAffinityKeyCookieDecryptionFailed = LoggerMessage.Define( + private static readonly Action _requestAffinityKeyDecryptionFailed = LoggerMessage.Define( LogLevel.Error, - EventIds.RequestAffinityKeyCookieDecryptionFailed, - "The request affinity key cookie decryption failed."); + EventIds.RequestAffinityKeyDecryptionFailed, + "The request affinity key decryption failed."); - private static readonly Action _requestAffinityKeyCookieCannotBeDecodedFromBase64 = LoggerMessage.Define( - LogLevel.Error, - EventIds.RequestAffinityKeyCookieCannotBeDecodedFromBase64, - "The request affinity key cookie cannot be decoded from Base64 representation."); + private static readonly Action _destinationMatchingToAffinityKeyNotFound = LoggerMessage.Define( + LogLevel.Warning, + EventIds.DestinationMatchingToAffinityKeyNotFound, + "Destination matching to the request affinity key is not found on backend `{backnedId}`. Configured failure policy will be applied."); public static void AffinityCannotBeEstablishedBecauseNoDestinationsFound(ILogger logger, string backendId) { _affinityCannotBeEstablishedBecauseNoDestinationsFound(logger, backendId, null); } - public static void RequestAffinityKeyCookieDecryptionFailed(ILogger logger, Exception ex) + public static void RequestAffinityKeyDecryptionFailed(ILogger logger, Exception ex) { - _requestAffinityKeyCookieDecryptionFailed(logger, ex); + _requestAffinityKeyDecryptionFailed(logger, ex); } - public static void RequestAffinityKeyCookieCannotBeDecodedFromBase64(ILogger logger) + public static void DestinationMatchingToAffinityKeyNotFound(ILogger logger, string backendId) { - _requestAffinityKeyCookieCannotBeDecodedFromBase64(logger, null); + _destinationMatchingToAffinityKeyNotFound(logger, backendId, null); } } } diff --git a/test/ReverseProxy.Tests/Middleware/AffinitizeRequestMiddlewareTests.cs b/test/ReverseProxy.Tests/Middleware/AffinitizeRequestMiddlewareTests.cs index 15bfd3a59..3cec79135 100644 --- a/test/ReverseProxy.Tests/Middleware/AffinitizeRequestMiddlewareTests.cs +++ b/test/ReverseProxy.Tests/Middleware/AffinitizeRequestMiddlewareTests.cs @@ -63,8 +63,7 @@ public async Task Invoke_MultipleCandidateDestinations_ChooseOneAndInvokeAffinit GetProviderForRandomDestination(expectedMode, Destinations, p => invokedMode = p.Mode) }; var nextInvoked = false; - var logger = new Mock>(); - logger.Setup(l => l.IsEnabled(LogLevel.Warning)).Returns(true); + var logger = AffinityTestHelper.GetLogger(); var middleware = new AffinitizeRequestMiddleware(c => { nextInvoked = true; return Task.CompletedTask; @@ -98,8 +97,7 @@ public async Task Invoke_NoDestinationChosen_LogWarningAndCallNext() { var backend = GetBackend(); var nextInvoked = false; - var logger = new Mock>(); - logger.Setup(l => l.IsEnabled(LogLevel.Warning)).Returns(true); + var logger = AffinityTestHelper.GetLogger(); var middleware = new AffinitizeRequestMiddleware(c => { nextInvoked = true; return Task.CompletedTask; diff --git a/test/ReverseProxy.Tests/Middleware/AffinitizedDestinationLookupMiddlewareTests.cs b/test/ReverseProxy.Tests/Middleware/AffinitizedDestinationLookupMiddlewareTests.cs index b93d1c4fe..e425213ab 100644 --- a/test/ReverseProxy.Tests/Middleware/AffinitizedDestinationLookupMiddlewareTests.cs +++ b/test/ReverseProxy.Tests/Middleware/AffinitizedDestinationLookupMiddlewareTests.cs @@ -68,7 +68,7 @@ public async Task Invoke_SuccessfulFlow_CallNext(AffinityStatus status, string f [InlineData(AffinityStatus.DestinationNotFound, false)] [InlineData(AffinityStatus.AffinityKeyExtractionFailed, true)] [InlineData(AffinityStatus.AffinityKeyExtractionFailed, false)] - public async Task Invoke_ErrorFlow_CallFailurePolicy(AffinityStatus affinityStatus, bool handled) + public async Task Invoke_ErrorFlow_CallFailurePolicy(AffinityStatus affinityStatus, bool keepProcessing) { var backend = GetBackend(); var providers = RegisterAffinityProviders(true, Destinations, backend.BackendId, ("Mode-B", affinityStatus, null, _ => { })); @@ -77,15 +77,16 @@ public async Task Invoke_ErrorFlow_CallFailurePolicy(AffinityStatus affinityStat var failurePolicies = RegisterFailurePolicies( affinityStatus, ("Policy-0", false, p => throw new InvalidOperationException($"Policy {p.Name} call is not expected.")), - (expectedPolicy, handled, p => invokedPolicy = p.Name)); + (expectedPolicy, keepProcessing, p => invokedPolicy = p.Name)); var nextInvoked = false; + var logger = AffinityTestHelper.GetLogger(); var middleware = new AffinitizedDestinationLookupMiddleware(c => { nextInvoked = true; return Task.CompletedTask; }, providers.Select(p => p.Object), failurePolicies.Select(p => p.Object), GetOperationLogger(true), - new Mock>().Object); + logger.Object); var context = new DefaultHttpContext(); context.Features.Set(backend); var destinationFeature = GetDestinationsFeature(Destinations); @@ -94,10 +95,16 @@ public async Task Invoke_ErrorFlow_CallFailurePolicy(AffinityStatus affinityStat await middleware.Invoke(context); Assert.Equal(expectedPolicy, invokedPolicy); - Assert.Equal(handled, nextInvoked); + Assert.Equal(keepProcessing, nextInvoked); failurePolicies[0].VerifyGet(p => p.Name, Times.Once); failurePolicies[0].VerifyNoOtherCalls(); failurePolicies[1].VerifyAll(); + if (!keepProcessing) + { + logger.Verify( + l => l.Log(LogLevel.Warning, EventIds.AffinityResolutionFailedForBackend, It.IsAny(), null, (Func)It.IsAny()), + Times.Once); + } } private IOperationLogger GetOperationLogger(bool callFailurePolicy) diff --git a/test/ReverseProxy.Tests/Service/SessionAffinity/AffinityTestHelper.cs b/test/ReverseProxy.Tests/Service/SessionAffinity/AffinityTestHelper.cs index 02cb3879b..14e4dfdd1 100644 --- a/test/ReverseProxy.Tests/Service/SessionAffinity/AffinityTestHelper.cs +++ b/test/ReverseProxy.Tests/Service/SessionAffinity/AffinityTestHelper.cs @@ -13,7 +13,9 @@ public static class AffinityTestHelper { public static Mock> GetLogger() { - return new Mock>(); + var result = new Mock>(); + result.Setup(l => l.IsEnabled(It.IsAny())).Returns(true); + return result; } public static Mock GetDataProtector() diff --git a/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTests.cs b/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTests.cs index 306773244..9ee81403c 100644 --- a/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTests.cs +++ b/test/ReverseProxy.Tests/Service/SessionAffinity/BaseSesstionAffinityProviderTests.cs @@ -10,12 +10,11 @@ using Microsoft.Extensions.Logging; using Microsoft.ReverseProxy.RuntimeModel; using Moq; -using Tests.Common; using Xunit; namespace Microsoft.ReverseProxy.Service.SessionAffinity { - public class BaseSesstionAffinityProviderTests : TestAutoMockBase + public class BaseSesstionAffinityProviderTests { private const string InvalidKeyNull = "!invalid key - null!"; private const string InvalidKeyThrow = "!invalid key - throw!"; @@ -30,10 +29,13 @@ public void Request_FindAffinitizedDestinations( AffinityStatus expectedStatus, DestinationInfo expectedDestination, byte[] expectedEncryptedKey, - bool unprotectCalled) + bool unprotectCalled, + LogLevel? expectedLogLevel, + EventId expectedEventId) { var dataProtector = GetDataProtector(); - var provider = new ProviderStub(dataProtector.Object, Mock().Object); + var logger = AffinityTestHelper.GetLogger>(); + var provider = new ProviderStub(dataProtector.Object, logger.Object); var affinityResult = provider.FindAffinitizedDestinations(context, allDestinations, "backend-1", _defaultOptions); if(unprotectCalled) @@ -43,6 +45,14 @@ public void Request_FindAffinitizedDestinations( Assert.Equal(expectedStatus, affinityResult.Status); Assert.Same(expectedDestination, affinityResult.Destinations?.FirstOrDefault()); + + if (expectedLogLevel != null) + { + logger.Verify( + l => l.Log(expectedLogLevel.Value, expectedEventId, It.IsAny(), It.IsAny(), (Func)It.IsAny()), + Times.Once); + } + if (expectedDestination != null) { Assert.Equal(1, affinityResult.Destinations.Count); @@ -56,7 +66,7 @@ public void Request_FindAffinitizedDestinations( [Fact] public void FindAffinitizedDestination_AffinityDisabledOnBackend_ReturnsAffinityDisabled() { - var provider = new ProviderStub(GetDataProtector().Object, Mock().Object); + var provider = new ProviderStub(GetDataProtector().Object, AffinityTestHelper.GetLogger>().Object); var options = new BackendConfig.BackendSessionAffinityOptions(false, _defaultOptions.Mode, _defaultOptions.AffinityFailurePolicy, _defaultOptions.Settings); Assert.Throws(() => provider.FindAffinitizedDestinations(new DefaultHttpContext(), new[] { new DestinationInfo("1") }, "backend-1", options)); } @@ -65,7 +75,7 @@ public void FindAffinitizedDestination_AffinityDisabledOnBackend_ReturnsAffinity public void AffinitizeRequest_AffinitiDisabled_DoNothing() { var dataProtector = GetDataProtector(); - var provider = new ProviderStub(dataProtector.Object, Mock().Object); + var provider = new ProviderStub(dataProtector.Object, AffinityTestHelper.GetLogger>().Object); Assert.Throws(() => provider.AffinitizeRequest(new DefaultHttpContext(), default, new DestinationInfo("id"))); } @@ -73,7 +83,7 @@ public void AffinitizeRequest_AffinitiDisabled_DoNothing() public void AffinitizeRequest_RequestIsAffinitized_DoNothing() { var dataProtector = GetDataProtector(); - var provider = new ProviderStub(dataProtector.Object, Mock().Object); + var provider = new ProviderStub(dataProtector.Object, AffinityTestHelper.GetLogger>().Object); var context = new DefaultHttpContext(); provider.DirectlySetExtractedKeyOnContext(context, "ExtractedKey"); provider.AffinitizeRequest(context, _defaultOptions, new DestinationInfo("id")); @@ -85,7 +95,7 @@ public void AffinitizeRequest_RequestIsAffinitized_DoNothing() public void AffinitizeRequest_RequestIsNotAffinitized_SetAffinityKey() { var dataProtector = GetDataProtector(); - var provider = new ProviderStub(dataProtector.Object, Mock().Object); + var provider = new ProviderStub(dataProtector.Object, AffinityTestHelper.GetLogger>().Object); var destination = new DestinationInfo("dest-A"); provider.AffinitizeRequest(new DefaultHttpContext(), _defaultOptions, destination); Assert.Equal("ZGVzdC1B", provider.LastSetEncryptedKey); @@ -96,7 +106,7 @@ public void AffinitizeRequest_RequestIsNotAffinitized_SetAffinityKey() [Fact] public void FindAffinitizedDestinations_AffinityOptionSettingNotFound_Throw() { - var provider = new ProviderStub(GetDataProtector().Object, Mock().Object); + var provider = new ProviderStub(GetDataProtector().Object, AffinityTestHelper.GetLogger>().Object); var options = GetOptionsWithUnknownSetting(); Assert.Throws(() => provider.FindAffinitizedDestinations(new DefaultHttpContext(), new[] { new DestinationInfo("dest-A") }, "backend-1", options)); } @@ -104,7 +114,7 @@ public void FindAffinitizedDestinations_AffinityOptionSettingNotFound_Throw() [Fact] public void AffinitizeRequest_AffinityOptionSettingNotFound_Throw() { - var provider = new ProviderStub(GetDataProtector().Object, Mock().Object); + var provider = new ProviderStub(GetDataProtector().Object, AffinityTestHelper.GetLogger>().Object); var options = GetOptionsWithUnknownSetting(); Assert.Throws(() => provider.AffinitizeRequest(new DefaultHttpContext(), options, new DestinationInfo("dest-A"))); } @@ -112,21 +122,22 @@ public void AffinitizeRequest_AffinityOptionSettingNotFound_Throw() [Fact] public void Ctor_MandatoryArgumentIsNull_Throw() { - Assert.Throws(() => new ProviderStub(null, Mock().Object)); + Assert.Throws(() => new ProviderStub(null, new Mock().Object)); // CreateDataProtector will return null - Assert.Throws(() => new ProviderStub(Mock().Object, Mock().Object)); + Assert.Throws(() => new ProviderStub(new Mock().Object, new Mock().Object)); Assert.Throws(() => new ProviderStub(GetDataProtector().Object, null)); } public static IEnumerable FindAffinitizedDestinationsCases() { var destinations = new[] { new DestinationInfo("dest-A"), new DestinationInfo("dest-B"), new DestinationInfo("dest-C") }; - yield return new object[] { GetHttpContext(new[] { ("SomeKey", "SomeValue") }), destinations, AffinityStatus.AffinityKeyNotSet, null, null, false }; - yield return new object[] { GetHttpContext(new[] { (KeyName, "dest-B") }), destinations, AffinityStatus.OK, destinations[1], Encoding.UTF8.GetBytes("dest-B"), true }; - yield return new object[] { GetHttpContext(new[] { (KeyName, "dest-Z") }), destinations, AffinityStatus.DestinationNotFound, null, Encoding.UTF8.GetBytes("dest-Z"), true }; - yield return new object[] { GetHttpContext(new[] { (KeyName, "dest-B") }), new DestinationInfo[0], AffinityStatus.DestinationNotFound, null, Encoding.UTF8.GetBytes("dest-B"), true }; - yield return new object[] { GetHttpContext(new[] { (KeyName, InvalidKeyNull) }), destinations, AffinityStatus.AffinityKeyExtractionFailed, null, Encoding.UTF8.GetBytes(InvalidKeyNull), true }; - yield return new object[] { GetHttpContext(new[] { (KeyName, InvalidKeyThrow) }), destinations, AffinityStatus.AffinityKeyExtractionFailed, null, Encoding.UTF8.GetBytes(InvalidKeyThrow), true }; + yield return new object[] { GetHttpContext(new[] { ("SomeKey", "SomeValue") }), destinations, AffinityStatus.AffinityKeyNotSet, null, null, false, null, null }; + yield return new object[] { GetHttpContext(new[] { (KeyName, "dest-B") }), destinations, AffinityStatus.OK, destinations[1], Encoding.UTF8.GetBytes("dest-B"), true, null, null }; + yield return new object[] { GetHttpContext(new[] { (KeyName, "dest-Z") }), destinations, AffinityStatus.DestinationNotFound, null, Encoding.UTF8.GetBytes("dest-Z"), true, LogLevel.Warning, EventIds.DestinationMatchingToAffinityKeyNotFound }; + yield return new object[] { GetHttpContext(new[] { (KeyName, "dest-B") }), new DestinationInfo[0], AffinityStatus.DestinationNotFound, null, Encoding.UTF8.GetBytes("dest-B"), true, LogLevel.Warning, EventIds.AffinityCannotBeEstablishedBecauseNoDestinationsFoundOnBackend }; + yield return new object[] { GetHttpContext(new[] { (KeyName, "/////") }, false), destinations, AffinityStatus.AffinityKeyExtractionFailed, null, Encoding.UTF8.GetBytes(InvalidKeyNull), false, LogLevel.Error, EventIds.RequestAffinityKeyDecryptionFailed }; + yield return new object[] { GetHttpContext(new[] { (KeyName, InvalidKeyNull) }), destinations, AffinityStatus.AffinityKeyExtractionFailed, null, Encoding.UTF8.GetBytes(InvalidKeyNull), true, LogLevel.Error, EventIds.RequestAffinityKeyDecryptionFailed }; + yield return new object[] { GetHttpContext(new[] { (KeyName, InvalidKeyThrow) }), destinations, AffinityStatus.AffinityKeyExtractionFailed, null, Encoding.UTF8.GetBytes(InvalidKeyThrow), true, LogLevel.Error, EventIds.RequestAffinityKeyDecryptionFailed }; } private static BackendConfig.BackendSessionAffinityOptions GetOptionsWithUnknownSetting() @@ -134,18 +145,18 @@ private static BackendConfig.BackendSessionAffinityOptions GetOptionsWithUnknown return new BackendConfig.BackendSessionAffinityOptions(true, "Stub", "Return503", new Dictionary { { "Unknown", "ZZZ" } }); } - private static HttpContext GetHttpContext((string Key, string Value)[] items) + private static HttpContext GetHttpContext((string Key, string Value)[] items, bool encodeToBase64 = true) { var context = new DefaultHttpContext { - Items = items.ToDictionary(i => (object)i.Key, i => (object)Convert.ToBase64String(Encoding.UTF8.GetBytes(i.Value))) + Items = items.ToDictionary(i => (object)i.Key, i => encodeToBase64 ? (object)Convert.ToBase64String(Encoding.UTF8.GetBytes(i.Value)) : i.Value) }; return context; } private Mock GetDataProtector() { - var result = Mock(); + var result = new Mock(); var nullBytes = Encoding.UTF8.GetBytes(InvalidKeyNull); var throwBytes = Encoding.UTF8.GetBytes(InvalidKeyThrow); result.Setup(p => p.Protect(It.IsAny())).Returns((byte[] k) => k); From 0285342fd22244983c37a2c9cc9e4f4934fc9432 Mon Sep 17 00:00:00 2001 From: Alexander Nikolaev Date: Thu, 4 Jun 2020 14:05:00 +0200 Subject: [PATCH 25/30] Unnecessary using removed --- .../Service/SessionAffinity/BaseSessionAffinityProvider.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs index 8c52fca92..37aac3673 100644 --- a/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.Text; using Microsoft.AspNetCore.DataProtection; using Microsoft.AspNetCore.Http; From 1325732bba82233338e102444ab1d093133efc2c Mon Sep 17 00:00:00 2001 From: Alexander Nikolaev Date: Thu, 4 Jun 2020 14:33:21 +0200 Subject: [PATCH 26/30] SessionAffinity sample scenario --- samples/SampleClient/Program.cs | 5 +- .../Scenarios/SessionAffinityScenario.cs | 84 +++++++++++++++++++ 2 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 samples/SampleClient/Scenarios/SessionAffinityScenario.cs diff --git a/samples/SampleClient/Program.cs b/samples/SampleClient/Program.cs index cb9690e10..f79d20941 100644 --- a/samples/SampleClient/Program.cs +++ b/samples/SampleClient/Program.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT License. using System; @@ -34,7 +34,8 @@ public static async Task Main(string[] args) {"Http1", () => new Http1Scenario()}, {"Http2", () => new Http2Scenario()}, {"RawUpgrade", () => new RawUpgradeScenario()}, - {"WebSockets", () => new WebSocketsScenario()} + {"WebSockets", () => new WebSocketsScenario()}, + {"SessionAffinity", () => new SessionAffinityScenario()} }; if (string.IsNullOrEmpty(parsedArgs.Scenario)) diff --git a/samples/SampleClient/Scenarios/SessionAffinityScenario.cs b/samples/SampleClient/Scenarios/SessionAffinityScenario.cs new file mode 100644 index 000000000..77ba4e2fe --- /dev/null +++ b/samples/SampleClient/Scenarios/SessionAffinityScenario.cs @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; + +namespace SampleClient.Scenarios +{ + internal class SessionAffinityScenario : IScenario + { + public async Task ExecuteAsync(CommandLineArgs args, CancellationToken cancellation) + { + using var handler = new HttpClientHandler + { + AllowAutoRedirect = false, + AutomaticDecompression = DecompressionMethods.None, + // Session affinity key will be stored in a cookie + UseCookies = true, + UseProxy = false + }; + using var client = new HttpMessageInvoker(handler); + var targetUri = new Uri(new Uri(args.Target, UriKind.Absolute), "api/dump"); + var stopwatch = Stopwatch.StartNew(); + + var request0 = new HttpRequestMessage(HttpMethod.Get, targetUri) { Version = new Version(1, 1) }; + Console.WriteLine($"Sending first request to {targetUri} with HTTP/1.1"); + var response0 = await client.SendAsync(request0, cancellation); + + PrintDuration(stopwatch, response0); + PrintAffinityCookie(handler, targetUri, response0); + await ReadAndPrintBody(response0, cancellation); + + stopwatch.Reset(); + + var request1 = new HttpRequestMessage(HttpMethod.Get, targetUri) { Version = new Version(1, 1) }; + Console.WriteLine($"Sending second request to {targetUri} with HTTP/1.1"); + var response1 = await client.SendAsync(request1, cancellation); + + PrintDuration(stopwatch, response1); + PrintAffinityCookie(handler, targetUri, response1); + await ReadAndPrintBody(response1, cancellation); + } + + private static void PrintDuration(Stopwatch stopwatch, HttpResponseMessage response) + { + Console.WriteLine($"Received response: {(int)response.StatusCode} in {stopwatch.ElapsedMilliseconds} ms"); + response.EnsureSuccessStatusCode(); + } + + private static void PrintAffinityCookie(HttpClientHandler handler, Uri targetUri, HttpResponseMessage response) + { + if (response.Headers.TryGetValues("Set-Cookie", out var setCookieValue)) + { + Console.WriteLine($"Received header Set-Cookie: {setCookieValue.ToArray()[0]}"); + } + else + { + Console.WriteLine($"Response doesn't have Set-Cookie header."); + } + + var affinityCookie = handler.CookieContainer.GetCookies(targetUri)[".Microsoft.ReverseProxy.Affinity"]; + Console.WriteLine($"Affinity key stored on a cookie {affinityCookie.Value}"); + } + + private static async Task ReadAndPrintBody(HttpResponseMessage response, CancellationToken cancellation) + { + var body = await response.Content.ReadAsStringAsync(cancellation); + var json = JsonDocument.Parse(body); + Console.WriteLine( + "Received response:" + + $"{Environment.NewLine}" + + $"{JsonSerializer.Serialize(json.RootElement, new JsonSerializerOptions { WriteIndented = true })}"); + response.EnsureSuccessStatusCode(); + } + } +} From ed7fc0f8796acd129b02f422b7fddcda52fd6132 Mon Sep 17 00:00:00 2001 From: Alexander Nikolaev Date: Mon, 8 Jun 2020 16:07:53 +0200 Subject: [PATCH 27/30] - Defaults for SessionAffinityOptions removed - DynamicConfigBuilder directly reads the session affinity constants - SessionAffinityOptions.Enabled added - AffinityFailurePolicies throw exceptions if invoked for a successful status - Wording fixed - More debug logging --- samples/ReverseProxy.Sample/Startup.cs | 1 - samples/ReverseProxy.Sample/appsettings.json | 1 + .../Contract/SessionAffinityDefaultOptions.cs | 40 --------- .../Contract/SessionAffinityOptions.cs | 6 ++ .../IReverseProxyBuilderExtensions.cs | 7 -- ...ReverseProxyServiceCollectionExtensions.cs | 2 + src/ReverseProxy/EventIds.cs | 2 + .../AffinitizedDestinationLookupMiddleware.cs | 90 ++++++++++++------- .../Service/Config/DynamicConfigBuilder.cs | 22 ++--- .../Management/ReverseProxyConfigManager.cs | 2 +- .../CustomHeaderSessionAffinityProvider.cs | 18 +++- .../SessionAffinity/IAffinityFailurePolicy.cs | 4 +- .../RedistributeAffinityFailurePolicy.cs | 7 ++ .../Return503ErrorAffinityFailurePolicy.cs | 4 +- .../SessionAffinityMiddlewareHelper.cs | 6 +- ...ustomHeaderSessionAffinityProviderTests.cs | 22 +++-- .../RedistributeAffinityFailurePolicyTests.cs | 11 +++ ...eturn503ErrorAffinityFailurePolicyTests.cs | 6 +- 18 files changed, 136 insertions(+), 115 deletions(-) delete mode 100644 src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityDefaultOptions.cs diff --git a/samples/ReverseProxy.Sample/Startup.cs b/samples/ReverseProxy.Sample/Startup.cs index e5eb97b56..42f9aa178 100644 --- a/samples/ReverseProxy.Sample/Startup.cs +++ b/samples/ReverseProxy.Sample/Startup.cs @@ -29,7 +29,6 @@ public Startup(IConfiguration configuration) public void ConfigureServices(IServiceCollection services) { services.AddControllers(); - services.AddDataProtection(); services.AddReverseProxy() .LoadFromConfig(_configuration.GetSection("ReverseProxy")) .AddProxyConfigFilter(); diff --git a/samples/ReverseProxy.Sample/appsettings.json b/samples/ReverseProxy.Sample/appsettings.json index d2af0dc1f..05e80defd 100644 --- a/samples/ReverseProxy.Sample/appsettings.json +++ b/samples/ReverseProxy.Sample/appsettings.json @@ -21,6 +21,7 @@ "Mode": "Random" }, "SessionAffinity": { + "Enabled": "true", "Mode": "Cookie" }, "Metadata": { diff --git a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityDefaultOptions.cs b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityDefaultOptions.cs deleted file mode 100644 index 4fa7c780c..000000000 --- a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityDefaultOptions.cs +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System; - -namespace Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract -{ - /// - /// Sets default values for session affinity configuration settings. - /// - public class SessionAffinityDefaultOptions - { - private string _defaultMode = SessionAffinityConstants.Modes.Cookie; - private string _defaultAffinityFailurePolicy = SessionAffinityConstants.AffinityFailurePolicies.Redistribute; - - /// - /// Default session affinity mode to be used when none is specified for a backend. - /// - public string DefaultMode - { - get => _defaultMode; - set => _defaultMode = value ?? throw new ArgumentNullException(nameof(value)); - } - - /// - /// Default affinity failure handling policy. - /// - public string AffinityFailurePolicy - { - get => _defaultAffinityFailurePolicy; - set => _defaultAffinityFailurePolicy = value ?? throw new ArgumentNullException(nameof(value)); - } - - /// - /// If set to enables session affinity for all backends using the default settings - /// which can be ovewritten by backend's configuration section. - /// - public bool EnabledForAllBackends { get; set; } - } -} diff --git a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityOptions.cs b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityOptions.cs index 300a0a099..11212c7cc 100644 --- a/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityOptions.cs +++ b/src/ReverseProxy/Abstractions/BackendDiscovery/Contract/SessionAffinityOptions.cs @@ -11,6 +11,11 @@ namespace Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract /// public sealed class SessionAffinityOptions { + /// + /// Indicates whether session affinity is enabled. + /// + public bool Enabled { get; set; } + /// /// Session affinity mode which is implemented by one of providers. /// @@ -30,6 +35,7 @@ internal SessionAffinityOptions DeepClone() { return new SessionAffinityOptions { + Enabled = Enabled, Mode = Mode, AffinityFailurePolicy = AffinityFailurePolicy, Settings = Settings?.DeepClone(StringComparer.Ordinal) diff --git a/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs b/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs index 0ad855f72..c50b0b1e7 100644 --- a/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs +++ b/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs @@ -90,13 +90,6 @@ public static IReverseProxyBuilder AddBackgroundWorkers(this IReverseProxyBuilde public static IReverseProxyBuilder AddSessionAffinityProvider(this IReverseProxyBuilder builder) { - return builder.AddSessionAffinityProvider(_ => { }); - } - - public static IReverseProxyBuilder AddSessionAffinityProvider(this IReverseProxyBuilder builder, Action configureOptions) - { - builder.Services.AddOptions().Configure(configureOptions); - builder.Services.TryAddEnumerable(new[] { new ServiceDescriptor(typeof(IAffinityFailurePolicy), typeof(RedistributeAffinityFailurePolicy), ServiceLifetime.Singleton), new ServiceDescriptor(typeof(IAffinityFailurePolicy), typeof(Return503ErrorAffinityFailurePolicy), ServiceLifetime.Singleton) diff --git a/src/ReverseProxy/Configuration/DependencyInjection/ReverseProxyServiceCollectionExtensions.cs b/src/ReverseProxy/Configuration/DependencyInjection/ReverseProxyServiceCollectionExtensions.cs index a4462253d..96cd6b64b 100644 --- a/src/ReverseProxy/Configuration/DependencyInjection/ReverseProxyServiceCollectionExtensions.cs +++ b/src/ReverseProxy/Configuration/DependencyInjection/ReverseProxyServiceCollectionExtensions.cs @@ -33,6 +33,8 @@ public static IReverseProxyBuilder AddReverseProxy(this IServiceCollection servi .AddProxy() .AddBackgroundWorkers(); + services.AddDataProtection(); + return builder; } diff --git a/src/ReverseProxy/EventIds.cs b/src/ReverseProxy/EventIds.cs index 502f8f732..c9645858b 100644 --- a/src/ReverseProxy/EventIds.cs +++ b/src/ReverseProxy/EventIds.cs @@ -46,5 +46,7 @@ internal static class EventIds public static readonly EventId NoDestinationOnBackendToEstablishRequestAffinity = new EventId(37, "NoDestinationOnBackendToEstablishRequestAffinity"); public static readonly EventId RequestAffinityKeyDecryptionFailed = new EventId(38, "RequestAffinityKeyDecryptionFailed"); public static readonly EventId DestinationMatchingToAffinityKeyNotFound = new EventId(39, "DestinationMatchingToAffinityKeyNotFound"); + public static readonly EventId RequestAffinityHeaderHasMultipleValues = new EventId(40, "RequestAffinityHeaderHasMultipleValues"); + public static readonly EventId AffinityResolutionFailureWasHandledProcessingWillBeContinued = new EventId(41, "AffinityResolutionFailureWasHandledProcessingWillBeContinued"); } } diff --git a/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs b/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs index 812b3de08..1ce82aaf1 100644 --- a/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs +++ b/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs @@ -7,6 +7,7 @@ using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; +using Microsoft.ReverseProxy.Abstractions; using Microsoft.ReverseProxy.Abstractions.Telemetry; using Microsoft.ReverseProxy.RuntimeModel; using Microsoft.ReverseProxy.Service.SessionAffinity; @@ -38,50 +39,61 @@ public AffinitizedDestinationLookupMiddleware( _affinityFailurePolicies = affinityFailurePolicies?.ToPolicyDictionary() ?? throw new ArgumentNullException(nameof(affinityFailurePolicies)); } - public async Task Invoke(HttpContext context) + public Task Invoke(HttpContext context) { var backend = context.GetRequiredBackend(); - var destinationsFeature = context.GetRequiredDestinationFeature(); - var destinations = destinationsFeature.Destinations; var options = backend.Config.Value?.SessionAffinityOptions ?? default; - if (options.Enabled) + if (!options.Enabled) { - var affinityResult = _operationLogger.Execute( + return _next(context); + } + + return InvokeInternal(context, options, backend); + } + + private async Task InvokeInternal(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, BackendInfo backend) + { + var destinationsFeature = context.GetRequiredDestinationFeature(); + var destinations = destinationsFeature.Destinations; + + var affinityResult = _operationLogger.Execute( "ReverseProxy.FindAffinitizedDestinations", - () => { + () => + { var currentProvider = _sessionAffinityProviders.GetRequiredServiceById(options.Mode); return currentProvider.FindAffinitizedDestinations(context, destinations, backend.BackendId, options); }); - switch (affinityResult.Status) - { - case AffinityStatus.OK: - destinationsFeature.Destinations = affinityResult.Destinations; - break; - case AffinityStatus.AffinityKeyNotSet: - // Nothing to do so just continue processing - break; - case AffinityStatus.AffinityKeyExtractionFailed: - case AffinityStatus.DestinationNotFound: - var keepProcessing = await _operationLogger.ExecuteAsync("ReverseProxy.HandleAffinityFailure", async () => - { - var failurePolicy = _affinityFailurePolicies.GetRequiredServiceById(options.AffinityFailurePolicy); - return await failurePolicy.Handle(context, options, affinityResult.Status); - }); - - if (!keepProcessing) - { - // Policy reported the failure is unrecoverable and took the full responsibility for its handling, - // so we simply stop processing. - Log.AffinityResolutionFailedForBackend(_logger, backend.BackendId); - return; - } - - break; - default: - throw new NotSupportedException($"Affinity status '{affinityResult.Status}' is not supported."); - } + switch (affinityResult.Status) + { + case AffinityStatus.OK: + destinationsFeature.Destinations = affinityResult.Destinations; + break; + case AffinityStatus.AffinityKeyNotSet: + // Nothing to do so just continue processing + break; + case AffinityStatus.AffinityKeyExtractionFailed: + case AffinityStatus.DestinationNotFound: + var keepProcessing = await _operationLogger.ExecuteAsync("ReverseProxy.HandleAffinityFailure", async () => + { + var failurePolicy = _affinityFailurePolicies.GetRequiredServiceById(options.AffinityFailurePolicy); + return await failurePolicy.Handle(context, options, affinityResult.Status); + }); + + if (!keepProcessing) + { + // Policy reported the failure is unrecoverable and took the full responsibility for its handling, + // so we simply stop processing. + Log.AffinityResolutionFailedForBackend(_logger, backend.BackendId); + return; + } + + Log.AffinityResolutionFailureWasHandledProcessingWillBeContinued(_logger, backend.BackendId, options.AffinityFailurePolicy); + + break; + default: + throw new NotSupportedException($"Affinity status '{affinityResult.Status}' is not supported."); } await _next(context); @@ -94,10 +106,20 @@ private static class Log EventIds.AffinityResolutionFailedForBackend, "Affinity resolution failed for backend `{backendId}`."); + private static readonly Action _affinityResolutionFailureWasHandledProcessingWillBeContinued = LoggerMessage.Define( + LogLevel.Debug, + EventIds.AffinityResolutionFailureWasHandledProcessingWillBeContinued, + "Affinity resolution failure for backend `{backendId}` was handled successfully by the policy `{policyName}`. Request processing will be continued."); + public static void AffinityResolutionFailedForBackend(ILogger logger, string backendId) { _affinityResolutioFailedForBackend(logger, backendId, null); } + + public static void AffinityResolutionFailureWasHandledProcessingWillBeContinued(ILogger logger, string backendId, string policyName) + { + _affinityResolutionFailureWasHandledProcessingWillBeContinued(logger, backendId, policyName, null); + } } } } diff --git a/src/ReverseProxy/Service/Config/DynamicConfigBuilder.cs b/src/ReverseProxy/Service/Config/DynamicConfigBuilder.cs index b5d861603..3213c67a1 100644 --- a/src/ReverseProxy/Service/Config/DynamicConfigBuilder.cs +++ b/src/ReverseProxy/Service/Config/DynamicConfigBuilder.cs @@ -22,7 +22,6 @@ internal class DynamicConfigBuilder : IDynamicConfigBuilder private readonly IRouteValidator _parsedRouteValidator; private readonly IDictionary _sessionAffinityProviders; private readonly IDictionary _affinityFailurePolicies; - private readonly SessionAffinityDefaultOptions _sessionAffinityDefaultOptions; public DynamicConfigBuilder( IEnumerable filters, @@ -30,8 +29,7 @@ public DynamicConfigBuilder( IRoutesRepo routesRepo, IRouteValidator parsedRouteValidator, IEnumerable sessionAffinityProviders, - IEnumerable affinityFailurePolicies, - IOptions sessionAffinityDefaultOptions) + IEnumerable affinityFailurePolicies) { Contracts.CheckValue(filters, nameof(filters)); Contracts.CheckValue(backendsRepo, nameof(backendsRepo)); @@ -45,7 +43,6 @@ public DynamicConfigBuilder( _parsedRouteValidator = parsedRouteValidator; _sessionAffinityProviders = sessionAffinityProviders.ToProviderDictionary(); _affinityFailurePolicies = affinityFailurePolicies.ToPolicyDictionary(); - _sessionAffinityDefaultOptions = sessionAffinityDefaultOptions?.Value ?? throw new ArgumentNullException(nameof(sessionAffinityDefaultOptions)); } public async Task> BuildConfigAsync(IConfigErrorReporter errorReporter, CancellationToken cancellation) @@ -100,22 +97,15 @@ public async Task> GetBackendsAsync(IConfigErrorRep private void ValidateSessionAffinity(IConfigErrorReporter errorReporter, string id, Backend backend) { - if (backend.SessionAffinity == null) + if (backend.SessionAffinity == null || !backend.SessionAffinity.Enabled) { - if (_sessionAffinityDefaultOptions.EnabledForAllBackends) - { - backend.SessionAffinity = new SessionAffinityOptions(); - } - else - { - // Session affinity is disabled - return; - } + // Session affinity is disabled + return; } if (string.IsNullOrEmpty(backend.SessionAffinity.Mode)) { - backend.SessionAffinity.Mode = _sessionAffinityDefaultOptions.DefaultMode; + backend.SessionAffinity.Mode = SessionAffinityConstants.Modes.Cookie; } var affinityMode = backend.SessionAffinity.Mode; @@ -126,7 +116,7 @@ private void ValidateSessionAffinity(IConfigErrorReporter errorReporter, string if (string.IsNullOrEmpty(backend.SessionAffinity.AffinityFailurePolicy)) { - backend.SessionAffinity.AffinityFailurePolicy = _sessionAffinityDefaultOptions.AffinityFailurePolicy; + backend.SessionAffinity.AffinityFailurePolicy = SessionAffinityConstants.AffinityFailurePolicies.Redistribute; } var affinityFailurePolicy = backend.SessionAffinity.AffinityFailurePolicy; diff --git a/src/ReverseProxy/Service/Management/ReverseProxyConfigManager.cs b/src/ReverseProxy/Service/Management/ReverseProxyConfigManager.cs index 925cfe147..30999d992 100644 --- a/src/ReverseProxy/Service/Management/ReverseProxyConfigManager.cs +++ b/src/ReverseProxy/Service/Management/ReverseProxyConfigManager.cs @@ -96,7 +96,7 @@ private void UpdateRuntimeBackends(DynamicConfigRoot config) new BackendConfig.BackendLoadBalancingOptions( mode: configBackend.LoadBalancing?.Mode ?? default), new BackendConfig.BackendSessionAffinityOptions( - enabled: configBackend.SessionAffinity != null, + enabled: configBackend.SessionAffinity?.Enabled ?? false, mode: configBackend.SessionAffinity?.Mode, affinityFailurePolicy: configBackend.SessionAffinity?.AffinityFailurePolicy, settings: configBackend.SessionAffinity?.Settings as IReadOnlyDictionary)); diff --git a/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs index bc36b3563..5e6faf644 100644 --- a/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/CustomHeaderSessionAffinityProvider.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System; using Microsoft.AspNetCore.DataProtection; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; @@ -12,6 +13,7 @@ namespace Microsoft.ReverseProxy.Service.SessionAffinity { internal class CustomHeaderSessionAffinityProvider : BaseSessionAffinityProvider { + public static readonly string DefaultCustomHeaderName = "X-Microsoft-Proxy-Affinity"; private const string CustomHeaderNameKey = "CustomHeaderName"; public CustomHeaderSessionAffinityProvider( @@ -29,7 +31,7 @@ protected override string GetDestinationAffinityKey(DestinationInfo destination) protected override (string Key, bool ExtractedSuccessfully) GetRequestAffinityKey(HttpContext context, in BackendConfig.BackendSessionAffinityOptions options) { - var customHeaderName = GetSettingValue(CustomHeaderNameKey, options); + var customHeaderName = options.Settings != null && options.Settings.TryGetValue(CustomHeaderNameKey, out var nameInSettings) ? nameInSettings : DefaultCustomHeaderName; var keyHeaderValues = context.Request.Headers[customHeaderName]; if (StringValues.IsNullOrEmpty(keyHeaderValues)) @@ -41,6 +43,7 @@ protected override (string Key, bool ExtractedSuccessfully) GetRequestAffinityKe if (keyHeaderValues.Count > 1) { // Multiple values is an ambiguous case which is considered a key extraction failure + Log.RequestAffinityHeaderHasMultipleValues(Logger, customHeaderName, keyHeaderValues.Count); return (Key: null, ExtractedSuccessfully: false); } @@ -52,5 +55,18 @@ protected override void SetAffinityKey(HttpContext context, in BackendConfig.Bac var customHeaderName = GetSettingValue(CustomHeaderNameKey, options); context.Response.Headers.Append(customHeaderName, Protect(unencryptedKey)); } + + private static class Log + { + private static readonly Action _requestAffinityHeaderHasMultipleValues = LoggerMessage.Define( + LogLevel.Error, + EventIds.RequestAffinityHeaderHasMultipleValues, + "The request affinity header `{headerName}` has `{valueCount}` values."); + + public static void RequestAffinityHeaderHasMultipleValues(ILogger logger, string headerName, int valueCount) + { + _requestAffinityHeaderHasMultipleValues(logger, headerName, valueCount, null); + } + } } } diff --git a/src/ReverseProxy/Service/SessionAffinity/IAffinityFailurePolicy.cs b/src/ReverseProxy/Service/SessionAffinity/IAffinityFailurePolicy.cs index 97f8a9dbe..a4ac902e4 100644 --- a/src/ReverseProxy/Service/SessionAffinity/IAffinityFailurePolicy.cs +++ b/src/ReverseProxy/Service/SessionAffinity/IAffinityFailurePolicy.cs @@ -26,8 +26,8 @@ internal interface IAffinityFailurePolicy /// Session affinity options set for the backend. /// Affinity resolution status. /// - /// if the failure has been considered recoverable and the request processing can proceed. - /// Otherwise, indicating that the request's processing must be terminated. + /// if the failure is considered recoverable and the request processing can proceed. + /// Otherwise, indicating that an error response has been generated and the request's processing must be terminated. /// public Task Handle(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, AffinityStatus affinityStatus); } diff --git a/src/ReverseProxy/Service/SessionAffinity/RedistributeAffinityFailurePolicy.cs b/src/ReverseProxy/Service/SessionAffinity/RedistributeAffinityFailurePolicy.cs index f04464d7b..316dd4b9b 100644 --- a/src/ReverseProxy/Service/SessionAffinity/RedistributeAffinityFailurePolicy.cs +++ b/src/ReverseProxy/Service/SessionAffinity/RedistributeAffinityFailurePolicy.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System; using System.Threading.Tasks; using Microsoft.AspNetCore.Http; using Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract; @@ -14,6 +15,12 @@ internal class RedistributeAffinityFailurePolicy : IAffinityFailurePolicy public Task Handle(HttpContext context, BackendConfig.BackendSessionAffinityOptions options, AffinityStatus affinityStatus) { + if (affinityStatus == AffinityStatus.OK + || affinityStatus == AffinityStatus.AffinityKeyNotSet) + { + throw new InvalidOperationException($"{nameof(RedistributeAffinityFailurePolicy)} is called to handle a successful request's affinity status {affinityStatus}."); + } + // Available destinations list have not been changed in the context, // so simply allow processing to proceed to load balancing. return Task.FromResult(true); diff --git a/src/ReverseProxy/Service/SessionAffinity/Return503ErrorAffinityFailurePolicy.cs b/src/ReverseProxy/Service/SessionAffinity/Return503ErrorAffinityFailurePolicy.cs index a8a001c69..40b2a1052 100644 --- a/src/ReverseProxy/Service/SessionAffinity/Return503ErrorAffinityFailurePolicy.cs +++ b/src/ReverseProxy/Service/SessionAffinity/Return503ErrorAffinityFailurePolicy.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System; using System.Threading.Tasks; using Microsoft.AspNetCore.Http; using Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract; @@ -17,8 +18,7 @@ public Task Handle(HttpContext context, BackendConfig.BackendSessionAffini if (affinityStatus == AffinityStatus.OK || affinityStatus == AffinityStatus.AffinityKeyNotSet) { - // We shouldn't get here, but allow the request to proceed further if that's the case. - return Task.FromResult(true); + throw new InvalidOperationException($"{nameof(Return503ErrorAffinityFailurePolicy)} is called to handle a successful request's affinity status {affinityStatus}."); } context.Response.StatusCode = 503; diff --git a/src/ReverseProxy/Service/SessionAffinity/SessionAffinityMiddlewareHelper.cs b/src/ReverseProxy/Service/SessionAffinity/SessionAffinityMiddlewareHelper.cs index f99cde986..738f88243 100644 --- a/src/ReverseProxy/Service/SessionAffinity/SessionAffinityMiddlewareHelper.cs +++ b/src/ReverseProxy/Service/SessionAffinity/SessionAffinityMiddlewareHelper.cs @@ -8,7 +8,7 @@ namespace Microsoft.ReverseProxy.Service.SessionAffinity { internal static class SessionAffinityMiddlewareHelper { - public static IDictionary ToDictionaryById(this IEnumerable services, Func idSelector) + public static IDictionary ToDictionaryByUniqueId(this IEnumerable services, Func idSelector) { if (services == null) { @@ -30,12 +30,12 @@ public static IDictionary ToDictionaryById(this IEnumerable ser public static IDictionary ToProviderDictionary(this IEnumerable sessionAffinityProviders) { - return ToDictionaryById(sessionAffinityProviders, p => p.Mode); + return ToDictionaryByUniqueId(sessionAffinityProviders, p => p.Mode); } public static IDictionary ToPolicyDictionary(this IEnumerable affinityFailurePolicies) { - return ToDictionaryById(affinityFailurePolicies, p => p.Name); + return ToDictionaryByUniqueId(affinityFailurePolicies, p => p.Name); } public static T GetRequiredServiceById(this IDictionary services, string id) diff --git a/test/ReverseProxy.Tests/Service/SessionAffinity/CustomHeaderSessionAffinityProviderTests.cs b/test/ReverseProxy.Tests/Service/SessionAffinity/CustomHeaderSessionAffinityProviderTests.cs index 00bc63ec1..53f2a21af 100644 --- a/test/ReverseProxy.Tests/Service/SessionAffinity/CustomHeaderSessionAffinityProviderTests.cs +++ b/test/ReverseProxy.Tests/Service/SessionAffinity/CustomHeaderSessionAffinityProviderTests.cs @@ -49,15 +49,21 @@ public void FindAffinitizedDestination_AffinityKeyIsSetOnRequest_Success() Assert.Same(affinitizedDestination, affinityResult.Destinations[0]); } - [Fact] - public void FindAffinitizedDestination_CustomHeaderNameIsNotSpecified_Throw() + [Theory] + [MemberData(nameof(FindAffinitizedDestination_CustomHeaderNameIsNotSpecified_Cases))] + public void FindAffinitizedDestination_CustomHeaderNameIsNotSpecified_UseDefaultName(Dictionary settings) { - var options = new BackendConfig.BackendSessionAffinityOptions(true, "CustomHeader", "Return503", null); + var options = new BackendConfig.BackendSessionAffinityOptions(true, "CustomHeader", "Return503", settings); var provider = new CustomHeaderSessionAffinityProvider(AffinityTestHelper.GetDataProtector().Object, AffinityTestHelper.GetLogger().Object); var context = new DefaultHttpContext(); - context.Request.Headers["SomeHeader"] = new[] { "SomeValue" }; + var affinitizedDestination = _destinations[1]; + context.Request.Headers[CustomHeaderSessionAffinityProvider.DefaultCustomHeaderName] = new[] { affinitizedDestination.DestinationId.ToUTF8BytesInBase64() }; - Assert.Throws(() => provider.FindAffinitizedDestinations(context, _destinations, "backend-1", options)); + var affinityResult = provider.FindAffinitizedDestinations(context, _destinations, "backend-1", options); + + Assert.Equal(AffinityStatus.OK, affinityResult.Status); + Assert.Equal(1, affinityResult.Destinations.Count); + Assert.Same(affinitizedDestination, affinityResult.Destinations[0]); } [Fact] @@ -91,5 +97,11 @@ public void AffinitizedRequest_AffinityKeyIsExtracted_DoNothing() Assert.False(context.Response.Headers.ContainsKey(AffinityHeaderName)); } + + public static IEnumerable FindAffinitizedDestination_CustomHeaderNameIsNotSpecified_Cases() + { + yield return new object[] { null }; + yield return new object[] { new Dictionary { { "SomeSetting", "SomeValue" } } }; + } } } diff --git a/test/ReverseProxy.Tests/Service/SessionAffinity/RedistributeAffinityFailurePolicyTests.cs b/test/ReverseProxy.Tests/Service/SessionAffinity/RedistributeAffinityFailurePolicyTests.cs index 3016347e7..dc0d85dc2 100644 --- a/test/ReverseProxy.Tests/Service/SessionAffinity/RedistributeAffinityFailurePolicyTests.cs +++ b/test/ReverseProxy.Tests/Service/SessionAffinity/RedistributeAffinityFailurePolicyTests.cs @@ -22,6 +22,17 @@ public async Task Handle_AnyAffinitStatus_ReturnTrue(AffinityStatus status) Assert.True(await policy.Handle(new DefaultHttpContext(), default, status)); } + [Theory] + [InlineData(AffinityStatus.OK)] + [InlineData(AffinityStatus.AffinityKeyNotSet)] + public async Task Handle_SuccessfulAffinityStatus_Throw(AffinityStatus status) + { + var policy = new RedistributeAffinityFailurePolicy(); + var context = new DefaultHttpContext(); + + await Assert.ThrowsAsync(() => policy.Handle(context, default, status)); + } + public static IEnumerable HandleCases() { // Successful statuses are also included because redistribute policy is not supposed to validate this parameter diff --git a/test/ReverseProxy.Tests/Service/SessionAffinity/Return503ErrorAffinityFailurePolicyTests.cs b/test/ReverseProxy.Tests/Service/SessionAffinity/Return503ErrorAffinityFailurePolicyTests.cs index f6335ae36..63d7ff383 100644 --- a/test/ReverseProxy.Tests/Service/SessionAffinity/Return503ErrorAffinityFailurePolicyTests.cs +++ b/test/ReverseProxy.Tests/Service/SessionAffinity/Return503ErrorAffinityFailurePolicyTests.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System; using System.Threading.Tasks; using Microsoft.AspNetCore.Http; using Microsoft.ReverseProxy.Abstractions.BackendDiscovery.Contract; @@ -27,13 +28,12 @@ public async Task Handle_FaultyAffinityStatus_RespondWith503(AffinityStatus stat [Theory] [InlineData(AffinityStatus.OK)] [InlineData(AffinityStatus.AffinityKeyNotSet)] - public async Task Handle_SuccessfulAffinityStatus_ReturnTrue(AffinityStatus status) + public async Task Handle_SuccessfulAffinityStatus_Throw(AffinityStatus status) { var policy = new Return503ErrorAffinityFailurePolicy(); var context = new DefaultHttpContext(); - Assert.True(await policy.Handle(context, default, status)); - Assert.Equal(200, context.Response.StatusCode); + await Assert.ThrowsAsync(() => policy.Handle(context, default, status)); } } } From d63c9f448939a20959b3b0622f06f699d8f5e4c4 Mon Sep 17 00:00:00 2001 From: Alexander Nikolaev Date: Mon, 8 Jun 2020 16:53:34 +0200 Subject: [PATCH 28/30] Fix RedistributeAffinityFailurePolicyTests --- .../RedistributeAffinityFailurePolicyTests.cs | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/test/ReverseProxy.Tests/Service/SessionAffinity/RedistributeAffinityFailurePolicyTests.cs b/test/ReverseProxy.Tests/Service/SessionAffinity/RedistributeAffinityFailurePolicyTests.cs index dc0d85dc2..5b47c1128 100644 --- a/test/ReverseProxy.Tests/Service/SessionAffinity/RedistributeAffinityFailurePolicyTests.cs +++ b/test/ReverseProxy.Tests/Service/SessionAffinity/RedistributeAffinityFailurePolicyTests.cs @@ -13,8 +13,9 @@ namespace Microsoft.ReverseProxy.Service.SessionAffinity public class RedistributeAffinityFailurePolicyTests { [Theory] - [MemberData(nameof(HandleCases))] - public async Task Handle_AnyAffinitStatus_ReturnTrue(AffinityStatus status) + [InlineData(AffinityStatus.AffinityKeyExtractionFailed)] + [InlineData(AffinityStatus.DestinationNotFound)] + public async Task Handle_FailedAffinityStatus_ReturnTrue(AffinityStatus status) { var policy = new RedistributeAffinityFailurePolicy(); @@ -32,15 +33,5 @@ public async Task Handle_SuccessfulAffinityStatus_Throw(AffinityStatus status) await Assert.ThrowsAsync(() => policy.Handle(context, default, status)); } - - public static IEnumerable HandleCases() - { - // Successful statuses are also included because redistribute policy is not supposed to validate this parameter - // and therefore must react properly to all affinity statuses. - foreach(AffinityStatus status in Enum.GetValues(typeof(AffinityStatus))) - { - yield return new object[] { status }; - } - } } } From 0e8e3822d957c9c10c5819148d9a77202560c244 Mon Sep 17 00:00:00 2001 From: Alexander Nikolaev Date: Mon, 8 Jun 2020 17:57:32 +0200 Subject: [PATCH 29/30] - Redundant destination assignment removed --- src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs b/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs index 3f3b7e86a..9b18dae56 100644 --- a/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs +++ b/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs @@ -59,11 +59,10 @@ public Task Invoke(HttpContext context) Log.MultipleDestinationsOnBackendToEstablishRequestAffinity(_logger, backend.BackendId); // It's assumed that all of them match to the request's affinity key. chosenDestination = candidateDestinations[_random.Next(candidateDestinations.Count)]; + destinationsFeature.Destinations = chosenDestination; } _operationLogger.Execute("ReverseProxy.AffinitizeRequest", () => AffinitizeRequest(context, options, chosenDestination)); - - destinationsFeature.Destinations = chosenDestination; } } From e4fee157b82d036aa47e3fba0ba680d42845af2a Mon Sep 17 00:00:00 2001 From: Alexander Nikolaev Date: Tue, 9 Jun 2020 12:24:49 +0200 Subject: [PATCH 30/30] Get rid of redundant async/await --- .../Middleware/AffinitizedDestinationLookupMiddleware.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs b/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs index 1ce82aaf1..ec5776614 100644 --- a/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs +++ b/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs @@ -75,10 +75,10 @@ private async Task InvokeInternal(HttpContext context, BackendConfig.BackendSess break; case AffinityStatus.AffinityKeyExtractionFailed: case AffinityStatus.DestinationNotFound: - var keepProcessing = await _operationLogger.ExecuteAsync("ReverseProxy.HandleAffinityFailure", async () => + var keepProcessing = await _operationLogger.ExecuteAsync("ReverseProxy.HandleAffinityFailure", () => { var failurePolicy = _affinityFailurePolicies.GetRequiredServiceById(options.AffinityFailurePolicy); - return await failurePolicy.Handle(context, options, affinityResult.Status); + return failurePolicy.Handle(context, options, affinityResult.Status); }); if (!keepProcessing)