Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

💡feat: allow for simpler override of Response header encoding of Forwarded Requests #2254

Merged
merged 11 commits into from
Sep 22, 2023
Merged
27 changes: 14 additions & 13 deletions docs/docfx/articles/config-files.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,10 @@ public Startup(IConfiguration configuration)
Configuration = configuration;
}

public void ConfigureServices(IServiceCollection services)
{
services.AddReverseProxy()
.LoadFromConfig(Configuration.GetSection("ReverseProxy"));
public void ConfigureServices(IServiceCollection services)
{
services.AddReverseProxy()
.LoadFromConfig(Configuration.GetSection("ReverseProxy"));
}

public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
Expand All @@ -27,11 +27,11 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
}

app.UseRouting();
app.UseEndpoints(endpoints =>
app.UseEndpoints(endpoints =>
{
endpoints.MapReverseProxy();
});
}
endpoints.MapReverseProxy();
});
}
```
**Note**: For details about middleware ordering see [here](https://docs.microsoft.com/aspnet/core/fundamentals/middleware/#middleware-order).

Expand Down Expand Up @@ -118,7 +118,7 @@ For additional fields see [ClusterConfig](xref:Yarp.ReverseProxy.Configuration.C
},
"ReverseProxy": {
// Routes tell the proxy which requests to forward
"Routes": {
"Routes": {
"minimumroute" : {
// Matches anything and routes it to www.example.com
"ClusterId": "minimumcluster",
Expand All @@ -134,7 +134,7 @@ For additional fields see [ClusterConfig](xref:Yarp.ReverseProxy.Configuration.C
"Authorization Policy" : "Anonymous", // Name of the policy or "Default", "Anonymous"
"CorsPolicy" : "Default", // Name of the CorsPolicy to apply to this route or "Default", "Disable"
"Match": {
"Path": "/something/{**remainder}", // The path to match using ASP.NET syntax.
"Path": "/something/{**remainder}", // The path to match using ASP.NET syntax.
"Hosts" : [ "www.aaaaa.com", "www.bbbbb.com"], // The host names to match, unspecified is any
"Methods" : [ "GET", "PUT" ], // The HTTP methods that match, uspecified is all
"Headers": [ // The headers to match, unspecified is any
Expand All @@ -161,7 +161,7 @@ For additional fields see [ClusterConfig](xref:Yarp.ReverseProxy.Configuration.C
{
"RequestHeader": "MyHeader",
"Set": "MyValue",
}
}
]
}
},
Expand Down Expand Up @@ -194,7 +194,7 @@ For additional fields see [ClusterConfig](xref:Yarp.ReverseProxy.Configuration.C
}
},
"HealthCheck": {
"Active": { // Makes API calls to validate the health.
"Active": { // Makes API calls to validate the health.
"Enabled": "true",
"Interval": "00:00:10",
"Timeout": "00:00:10",
Expand All @@ -212,7 +212,8 @@ For additional fields see [ClusterConfig](xref:Yarp.ReverseProxy.Configuration.C
"DangerousAcceptAnyServerCertificate" : false,
"MaxConnectionsPerServer" : 1024,
"EnableMultipleHttp2Connections" : true,
"RequestHeaderEncoding" : "Latin1" // How to interpret non ASCII characters in header values
"RequestHeaderEncoding" : "Latin1", // How to interpret non ASCII characters in request header values
"ResponseHeaderEncoding" : "Latin1" // How to interpret non ASCII characters in response header values
},
"HttpRequest" : { // Options for sending request to destination
"ActivityTimeout" : "00:02:00",
Expand Down
14 changes: 9 additions & 5 deletions docs/docfx/articles/http-client-config.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ HTTP client configuration is based on [HttpClientConfig](xref:Yarp.ReverseProxy.
"MaxConnectionsPerServer": "<int>",
"DangerousAcceptAnyServerCertificate": "<bool>",
"RequestHeaderEncoding": "<encoding-name>",
"ResponseHeaderEncoding": "<encoding-name>",
"EnableMultipleHttp2Connections": "<bool>"
"WebProxy": {
"Address": "<url>",
Expand All @@ -44,11 +45,15 @@ Configuration settings:
```JSON
"DangerousAcceptAnyServerCertificate": "true"
```
- RequestHeaderEncoding - enables other than ASCII encoding for outgoing request headers. Setting this value will leverage [`SocketsHttpHandler.RequestHeaderEncodingSelector`](https://docs.microsoft.com/dotnet/api/system.net.http.socketshttphandler.requestheaderencodingselector) and use the selected encoding for all headers. If you need more granular approach, please use custom `IProxyHttpClientFactory`. The value is then parsed by [`Encoding.GetEncoding`](https://docs.microsoft.com/dotnet/api/system.text.encoding.getencoding#System_Text_Encoding_GetEncoding_System_String_), use values like: "utf-8", "iso-8859-1", etc.
- RequestHeaderEncoding - enables other than ASCII encoding for outgoing request headers. Setting this value will leverage [`SocketsHttpHandler.RequestHeaderEncodingSelector`](https://docs.microsoft.com/dotnet/api/system.net.http.socketshttphandler.requestheaderencodingselector) and use the selected encoding for all headers. The value is then parsed by [`Encoding.GetEncoding`](https://docs.microsoft.com/dotnet/api/system.text.encoding.getencoding#System_Text_Encoding_GetEncoding_System_String_), use values like: "utf-8", "iso-8859-1", etc.
```JSON
"RequestHeaderEncoding": "utf-8"
```
If you're using an encoding other than ASCII (or UTF-8 for Kestrel) you also need to set your server to accept requests with such headers. For example, use [`KestrelServerOptions.RequestHeaderEncodingSelector`](https://docs.microsoft.com/dotnet/api/Microsoft.AspNetCore.Server.Kestrel.Core.KestrelServerOptions.RequestHeaderEncodingSelector) to set up Kestrel to accept Latin1 ("iso-8859-1") headers:
MihaZupan marked this conversation as resolved.
Show resolved Hide resolved
- ResponseHeaderEncoding - enables other than ASCII encoding for incoming response headers (from requests that the proxy would forward out). Setting this value will leverage [`SocketsHttpHandler.ResponseHeaderEncodingSelector`](https://docs.microsoft.com/dotnet/api/system.net.http.socketshttphandler.responseheaderencodingselector) and use the selected encoding for all headers. The value is then parsed by [`Encoding.GetEncoding`](https://docs.microsoft.com/dotnet/api/system.text.encoding.getencoding#System_Text_Encoding_GetEncoding_System_String_), use values like: "utf-8", "iso-8859-1", etc.
```JSON
"ResponseHeaderEncoding": "utf-8"
```
If you need more granular approach, please use custom `IForwarderHttpClientFactory`. However, if you're using an encoding other than ASCII (or UTF-8 for Kestrel) you also need to set your server to accept requests and/or send responses with such headers. For example, use [`KestrelServerOptions.RequestHeaderEncodingSelector`](https://docs.microsoft.com/dotnet/api/Microsoft.AspNetCore.Server.Kestrel.Core.KestrelServerOptions.RequestHeaderEncodingSelector) / [`.ResponseHeaderEncodingSelector`](https://docs.microsoft.com/dotnet/api/Microsoft.AspNetCore.Server.Kestrel.Core.KestrelServerOptions.ResponseHeaderEncodingSelector) to set up Kestrel to accept Latin1 ("iso-8859-1") headers:
MihaZupan marked this conversation as resolved.
Show resolved Hide resolved
MihaZupan marked this conversation as resolved.
Show resolved Hide resolved
```C#
private static IHostBuilder CreateHostBuilder(string[] args) =>
Host.CreateDefaultBuilder(args)
Expand All @@ -58,6 +63,8 @@ private static IHostBuilder CreateHostBuilder(string[] args) =>
.ConfigureKestrel(kestrel =>
{
kestrel.RequestHeaderEncodingSelector = _ => Encoding.Latin1;
// and/or
kestrel.ResponseHeaderEncodingSelector = _ => Encoding.Latin1;
});
});
```
Expand All @@ -77,9 +84,6 @@ private static IHostBuilder CreateHostBuilder(string[] args) =>
}
```

At the moment, there is no solution for changing encoding for response headers in Kestrel (see [aspnetcore#26334](https://github.com/dotnet/aspnetcore/issues/26334)), only ASCII is accepted.


### HttpRequest
HTTP request configuration is based on [ForwarderRequestConfig](xref:Yarp.ReverseProxy.Forwarder.ForwarderRequestConfig) and represented by the following configuration schema.
```JSON
Expand Down
7 changes: 4 additions & 3 deletions samples/ReverseProxy.Config.Sample/appsettings.json
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
"Authorization Policy": "Anonymous", // Name of the policy or "Default", "Anonymous"
"CorsPolicy": "disable", // Name of the CorsPolicy to apply to this route or "default", "disable"
"Match": { // Rules that have to be met for the route to match the request
"Path": "/download/{**remainder}", // The path to match using ASP.NET syntax.
"Path": "/download/{**remainder}", // The path to match using ASP.NET syntax.
"Hosts": [ "localhost", "www.aaaaa.com", "www.bbbbb.com" ], // The host names to match, unspecified is any
"Methods": [ "GET", "PUT" ], // The HTTP methods that match, unspecified is all
"Headers": [ // The headers to match, unspecified is any
Expand Down Expand Up @@ -91,7 +91,7 @@
"AffinityKeyName": "MySessionCookieName" // Required, no default
},
"HealthCheck": { // Ways to determine which destinations should be filtered out due to unhealthy state
"Active": { // Makes API calls to validate the health of each destination
"Active": { // Makes API calls to validate the health of each destination
"Enabled": "true",
"Interval": "00:00:10", // How often to query for health data
"Timeout": "00:00:10", // Timeout for the health check request/response
Expand All @@ -110,7 +110,8 @@
"DangerousAcceptAnyServerCertificate": true, // Disables destination cert validation
"MaxConnectionsPerServer": 1024, // Destination server can further limit this number
"EnableMultipleHttp2Connections": true,
"RequestHeaderEncoding": "Latin1" // How to interpret non ASCII characters in header values
"RequestHeaderEncoding": "Latin1", // How to interpret non ASCII characters in proxied request's header values
"ResponseHeaderEncoding": "Latin1" // How to interpret non ASCII characters in proxied request's response header values
},
"HttpRequest": { // Options for sending request to destination
"Timeout": "00:02:00", // Timeout for the HttpRequest
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,7 @@ private static RouteQueryParameter CreateRouteQueryParameter(IConfigurationSecti
MaxConnectionsPerServer = section.ReadInt32(nameof(HttpClientConfig.MaxConnectionsPerServer)),
EnableMultipleHttp2Connections = section.ReadBool(nameof(HttpClientConfig.EnableMultipleHttp2Connections)),
RequestHeaderEncoding = section[nameof(HttpClientConfig.RequestHeaderEncoding)],
ResponseHeaderEncoding = section[nameof(HttpClientConfig.ResponseHeaderEncoding)],
WebProxy = webProxy
};
}
Expand Down Expand Up @@ -377,7 +378,7 @@ private static DestinationConfig CreateDestination(IConfigurationSection section
Address = section[nameof(DestinationConfig.Address)]!,
Health = section[nameof(DestinationConfig.Health)],
Metadata = section.GetSection(nameof(DestinationConfig.Metadata)).ReadStringDictionary(),
Host = section[nameof(DestinationConfig.Host)]
Host = section[nameof(DestinationConfig.Host)]
};
}

Expand Down
21 changes: 17 additions & 4 deletions src/ReverseProxy/Configuration/ConfigValidator.cs
Original file line number Diff line number Diff line change
Expand Up @@ -455,16 +455,29 @@ private static void ValidateProxyHttpClient(IList<Exception> errors, ClusterConf
errors.Add(new ArgumentException($"Max connections per server limit set on the cluster '{cluster.ClusterId}' must be positive."));
}

var encoding = cluster.HttpClient.RequestHeaderEncoding;
if (encoding is not null)
var requestHeaderEncoding = cluster.HttpClient.RequestHeaderEncoding;
if (requestHeaderEncoding is not null)
{
try
{
Encoding.GetEncoding(encoding);
Encoding.GetEncoding(requestHeaderEncoding);
}
catch (ArgumentException aex)
{
errors.Add(new ArgumentException($"Invalid header encoding '{encoding}'.", aex));
errors.Add(new ArgumentException($"Invalid request header encoding '{requestHeaderEncoding}'.", aex));
}
}

var responseHeaderEncoding = cluster.HttpClient.ResponseHeaderEncoding;
if (responseHeaderEncoding is not null)
{
try
{
Encoding.GetEncoding(responseHeaderEncoding);
}
catch (ArgumentException aex)
{
errors.Add(new ArgumentException($"Invalid response header encoding '{responseHeaderEncoding}'.", aex));
}
}
}
Expand Down
9 changes: 8 additions & 1 deletion src/ReverseProxy/Configuration/HttpClientConfig.cs
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ public sealed record HttpClientConfig
public int? MaxConnectionsPerServer { get; init; }

/// <summary>
/// Optional web proxy used when communicating with the destination server.
/// Optional web proxy used when communicating with the destination server.
/// </summary>
public WebProxyConfig? WebProxy { get; init; }

Expand All @@ -49,6 +49,11 @@ public sealed record HttpClientConfig
/// </summary>
public string? RequestHeaderEncoding { get; init; }

/// <summary>
/// Enables non-ASCII header encoding for incoming responses.
/// </summary>
public string? ResponseHeaderEncoding { get; init; }

public bool Equals(HttpClientConfig? other)
{
if (other is null)
Expand All @@ -62,6 +67,7 @@ public bool Equals(HttpClientConfig? other)
&& EnableMultipleHttp2Connections == other.EnableMultipleHttp2Connections
// Comparing by reference is fine here since Encoding.GetEncoding returns the same instance for each encoding.
&& RequestHeaderEncoding == other.RequestHeaderEncoding
&& ResponseHeaderEncoding == other.ResponseHeaderEncoding
&& WebProxy == other.WebProxy;
}

Expand All @@ -72,6 +78,7 @@ public override int GetHashCode()
MaxConnectionsPerServer,
EnableMultipleHttp2Connections,
RequestHeaderEncoding,
ResponseHeaderEncoding,
WebProxy);
}
}
6 changes: 6 additions & 0 deletions src/ReverseProxy/Forwarder/ForwarderHttpClientFactory.cs
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,12 @@ protected virtual void ConfigureHandler(ForwarderHttpClientContext context, Sock
handler.RequestHeaderEncodingSelector = (_, _) => encoding;
}

if (newConfig.ResponseHeaderEncoding is not null)
{
var encoding = Encoding.GetEncoding(newConfig.ResponseHeaderEncoding);
handler.ResponseHeaderEncodingSelector = (_, _) => encoding;
}

var webProxy = TryCreateWebProxy(newConfig.WebProxy);
if (webProxy is not null)
{
Expand Down
6 changes: 4 additions & 2 deletions test/ReverseProxy.Tests/Configuration/ClusterConfigTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,8 @@ public void Equals_Same_Value_Returns_True()
SslProtocols = SslProtocols.Tls11 | SslProtocols.Tls12,
MaxConnectionsPerServer = 10,
DangerousAcceptAnyServerCertificate = true,
RequestHeaderEncoding = Encoding.UTF8.WebName
RequestHeaderEncoding = Encoding.UTF8.WebName,
ResponseHeaderEncoding = Encoding.UTF8.WebName
},
HttpRequest = new ForwarderRequestConfig
{
Expand Down Expand Up @@ -161,7 +162,8 @@ public void Equals_Same_Value_Returns_True()
SslProtocols = SslProtocols.Tls11 | SslProtocols.Tls12,
MaxConnectionsPerServer = 10,
DangerousAcceptAnyServerCertificate = true,
RequestHeaderEncoding = Encoding.UTF8.WebName
RequestHeaderEncoding = Encoding.UTF8.WebName,
ResponseHeaderEncoding = Encoding.UTF8.WebName
},
HttpRequest = new ForwarderRequestConfig
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,7 @@ public class ConfigurationConfigProviderTests
""MaxConnectionsPerServer"": 10,
""EnableMultipleHttp2Connections"": true,
""RequestHeaderEncoding"": ""utf-8"",
""ResponseHeaderEncoding"": ""utf-8"",
""WebProxy"": {
""Address"": ""http://localhost:8080"",
""BypassOnLocal"": true,
Expand Down Expand Up @@ -309,7 +310,7 @@ public class ConfigurationConfigProviderTests
}
},
""Routes"": {
""routeA"" : {
""routeA"" : {
""Match"": {
""Methods"": [
""GET"",
Expand Down Expand Up @@ -547,6 +548,7 @@ private void VerifyValidAbstractConfig(IProxyConfig validConfig, IProxyConfig ab
Assert.Equal(cluster1.HttpClient.MaxConnectionsPerServer, abstractCluster1.HttpClient.MaxConnectionsPerServer);
Assert.Equal(cluster1.HttpClient.EnableMultipleHttp2Connections, abstractCluster1.HttpClient.EnableMultipleHttp2Connections);
Assert.Equal(Encoding.UTF8.WebName, abstractCluster1.HttpClient.RequestHeaderEncoding);
Assert.Equal(Encoding.UTF8.WebName, abstractCluster1.HttpClient.ResponseHeaderEncoding);
Assert.Equal(SslProtocols.Tls11 | SslProtocols.Tls12, abstractCluster1.HttpClient.SslProtocols);
Assert.Equal(cluster1.HttpRequest.ActivityTimeout, abstractCluster1.HttpRequest.ActivityTimeout);
Assert.Equal(HttpVersion.Version10, abstractCluster1.HttpRequest.Version);
Expand Down
47 changes: 44 additions & 3 deletions test/ReverseProxy.Tests/Configuration/ConfigValidatorTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -1108,7 +1108,7 @@ public async Task SetAvailableDestinationsPolicy_Invalid()
}

[Fact]
public async Task HttpClient_HeaderEncoding_Valid()
public async Task HttpClient_RequestHeaderEncoding_Valid()
{
var services = CreateServices();
var validator = services.GetRequiredService<IConfigValidator>();
Expand All @@ -1128,7 +1128,7 @@ public async Task HttpClient_HeaderEncoding_Valid()
}

[Fact]
public async Task HttpClient_HeaderEncoding_Invalid()
public async Task HttpClient_RequestHeaderEncoding_Invalid()
{
var services = CreateServices();
var validator = services.GetRequiredService<IConfigValidator>();
Expand All @@ -1145,6 +1145,47 @@ public async Task HttpClient_HeaderEncoding_Invalid()
var errors = await validator.ValidateClusterAsync(cluster);

Assert.Equal(1, errors.Count);
Assert.Equal("Invalid header encoding 'base64'.", errors[0].Message);
Assert.Equal("Invalid request header encoding 'base64'.", errors[0].Message);
}

[Fact]
public async Task HttpClient_ResponseHeaderEncoding_Valid()
{
var services = CreateServices();
var validator = services.GetRequiredService<IConfigValidator>();

var cluster = new ClusterConfig
{
ClusterId = "cluster1",
HttpClient = new HttpClientConfig
{
ResponseHeaderEncoding = "utf-8"
}
};

var errors = await validator.ValidateClusterAsync(cluster);

Assert.Equal(0, errors.Count);
}

[Fact]
public async Task HttpClient_ResponseHeaderEncoding_Invalid()
{
var services = CreateServices();
var validator = services.GetRequiredService<IConfigValidator>();

var cluster = new ClusterConfig
{
ClusterId = "cluster1",
HttpClient = new HttpClientConfig
{
ResponseHeaderEncoding = "base64"
}
};

var errors = await validator.ValidateClusterAsync(cluster);

Assert.Equal(1, errors.Count);
Assert.Equal("Invalid response header encoding 'base64'.", errors[0].Message);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ public void Equals_Same_Value_Returns_True()
MaxConnectionsPerServer = 20,
WebProxy = new WebProxyConfig() { Address = new Uri("http://localhost:8080"), BypassOnLocal = true, UseDefaultCredentials = true },
RequestHeaderEncoding = Encoding.UTF8.WebName,
ResponseHeaderEncoding = Encoding.UTF8.WebName,
};

var options2 = new HttpClientConfig
Expand All @@ -29,6 +30,7 @@ public void Equals_Same_Value_Returns_True()
MaxConnectionsPerServer = 20,
WebProxy = new WebProxyConfig() { Address = new Uri("http://localhost:8080"), BypassOnLocal = true, UseDefaultCredentials = true },
RequestHeaderEncoding = Encoding.UTF8.WebName,
ResponseHeaderEncoding = Encoding.UTF8.WebName,
};

var equals = options1.Equals(options2);
Expand Down