From 2cea39b49b5392bc4d126814ceadf34a0bb7c0b4 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Tue, 30 Jan 2024 16:10:27 -0800 Subject: [PATCH] Rewrite HttpFaultInjector to use ILogger and built-in ASP.NET Core logging (#7476) * Rewrite failtinjector to use ILogger and built-in logging --- .../Azure.Sdk.Tools.HttpFaultInjector.csproj | 1 + .../FaultInjectingMiddleware.cs | 245 ++++++++++ .../Program.cs | 428 ++---------------- .../UpstreamResponse.cs | 43 ++ .../Utils.cs | 61 +++ .../appsettings.json | 11 + 6 files changed, 401 insertions(+), 388 deletions(-) create mode 100644 tools/http-fault-injector/Azure.Sdk.Tools.HttpFaultInjector/FaultInjectingMiddleware.cs create mode 100644 tools/http-fault-injector/Azure.Sdk.Tools.HttpFaultInjector/UpstreamResponse.cs create mode 100644 tools/http-fault-injector/Azure.Sdk.Tools.HttpFaultInjector/Utils.cs create mode 100644 tools/http-fault-injector/Azure.Sdk.Tools.HttpFaultInjector/appsettings.json diff --git a/tools/http-fault-injector/Azure.Sdk.Tools.HttpFaultInjector/Azure.Sdk.Tools.HttpFaultInjector.csproj b/tools/http-fault-injector/Azure.Sdk.Tools.HttpFaultInjector/Azure.Sdk.Tools.HttpFaultInjector.csproj index 10b1b225133..4de145d3572 100644 --- a/tools/http-fault-injector/Azure.Sdk.Tools.HttpFaultInjector/Azure.Sdk.Tools.HttpFaultInjector.csproj +++ b/tools/http-fault-injector/Azure.Sdk.Tools.HttpFaultInjector/Azure.Sdk.Tools.HttpFaultInjector.csproj @@ -12,5 +12,6 @@ + diff --git a/tools/http-fault-injector/Azure.Sdk.Tools.HttpFaultInjector/FaultInjectingMiddleware.cs b/tools/http-fault-injector/Azure.Sdk.Tools.HttpFaultInjector/FaultInjectingMiddleware.cs new file mode 100644 index 00000000000..25cecf94a8a --- /dev/null +++ b/tools/http-fault-injector/Azure.Sdk.Tools.HttpFaultInjector/FaultInjectingMiddleware.cs @@ -0,0 +1,245 @@ +using System.Threading.Tasks; +using System.Threading; +using Microsoft.Extensions.Logging; +using System; +using Microsoft.AspNetCore.Connections.Features; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Primitives; +using System.Collections.Generic; +using System.Net.Http; +using System.Net.Sockets; +using System.Reflection; +using System.Linq; +using System.IO; +using System.Buffers; +using System.Diagnostics; + +namespace Azure.Sdk.Tools.HttpFaultInjector +{ + public class FaultInjectingMiddleware + { + private readonly ILogger _logger; + private readonly HttpClient _httpClient; + public FaultInjectingMiddleware(RequestDelegate _, IHttpClientFactory httpClientFactory, ILogger logger) + { + _logger = logger; + _httpClient = httpClientFactory.CreateClient("upstream"); + } + + public async Task InvokeAsync(HttpContext context) + { + string faultHeaderValue = context.Request.Headers[Utils.ResponseSelectionHeader]; + string upstreamBaseUri = context.Request.Headers[Utils.UpstreamBaseUriHeader]; + + if (ValidateOrReadFaultMode(faultHeaderValue, out var fault)) + { + await ProxyResponse(context, upstreamBaseUri, fault, context.RequestAborted); + } + else + { + context.Response.StatusCode = 400; + } + } + + + private async Task SendUpstreamRequest(HttpRequest request, string uri, CancellationToken cancellationToken) + { + var incomingUriBuilder = new UriBuilder() + { + Scheme = request.Scheme, + Host = request.Host.Host, + Path = request.Path.Value, + Query = request.QueryString.Value, + }; + if (request.Host.Port.HasValue) + { + incomingUriBuilder.Port = request.Host.Port.Value; + } + var incomingUri = incomingUriBuilder.Uri; + + var upstreamUriBuilder = new UriBuilder(uri) + { + Path = request.Path.Value, + Query = request.QueryString.Value, + }; + + var upstreamUri = upstreamUriBuilder.Uri; + + using (var upstreamRequest = new HttpRequestMessage(new HttpMethod(request.Method), upstreamUri)) + { + if (request.ContentLength > 0) + { + upstreamRequest.Content = new StreamContent(request.Body); + foreach (var header in request.Headers.Where(h => Utils.ContentRequestHeaders.Contains(h.Key))) + { + upstreamRequest.Content.Headers.Add(header.Key, values: header.Value); + } + } + + foreach (var header in request.Headers.Where(h => !Utils.ExcludedRequestHeaders.Contains(h.Key) && !Utils.ContentRequestHeaders.Contains(h.Key))) + { + if (!upstreamRequest.Headers.TryAddWithoutValidation(header.Key, values: header.Value)) + { + throw new InvalidOperationException($"Could not add header {header.Key} with value {header.Value}"); + } + } + + var upstreamResponseMessage = await _httpClient.SendAsync(upstreamRequest); + var headers = new List>>(); + // Must skip "Transfer-Encoding" header, since if it's set manually Kestrel requires you to implement + // your own chunking. + headers.AddRange(upstreamResponseMessage.Headers.Where(header => !string.Equals(header.Key, "Transfer-Encoding", StringComparison.OrdinalIgnoreCase))); + headers.AddRange(upstreamResponseMessage.Content.Headers); + + var upstreamResponse = await UpstreamResponseFromHttpContent(upstreamResponseMessage.Content, cancellationToken); + upstreamResponse.StatusCode = (int)upstreamResponseMessage.StatusCode; + upstreamResponse.Headers = headers.Select(h => new KeyValuePair(h.Key, h.Value.ToArray())); + + return upstreamResponse; + } + } + + + private async Task UpstreamResponseFromHttpContent(HttpContent content, CancellationToken cancellationToken) + { + if (content.Headers.ContentLength == null) + { + MemoryStream contentStream = await BufferContentAsync(content, cancellationToken); + // we no longer need that content and can let the connection go back to the pool. + content.Dispose(); + return new UpstreamResponse(contentStream); + } + + return new UpstreamResponse(content); + } + + private async Task BufferContentAsync(HttpContent content, CancellationToken cancellationToken) + { + Debug.Assert(content.Headers.ContentLength == null, "We should not buffer content if length is available."); + + _logger.LogWarning("Response does not have content length (is chunked or malformed) and is being buffered"); + byte[] contentBytes = await content.ReadAsByteArrayAsync(cancellationToken); + _logger.LogInformation("Finished buffering response body ({length})", contentBytes.Length); + return new MemoryStream(contentBytes); + } + + private async Task ProxyResponse(HttpContext context, string upstreamUri, string fault, CancellationToken cancellationToken) + { + UpstreamResponse upstreamResponse = await SendUpstreamRequest(context.Request, upstreamUri, cancellationToken); + switch (fault) + { + case "f": + // Full response + await SendDownstreamResponse(context.Response, upstreamResponse, upstreamResponse.ContentLength, cancellationToken); + return; + case "p": + // Partial Response (full headers, 50% of body), then wait indefinitely + await SendDownstreamResponse(context.Response, upstreamResponse, upstreamResponse.ContentLength / 2, cancellationToken); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + return; + case "pc": + // Partial Response (full headers, 50% of body), then close (TCP FIN) + await SendDownstreamResponse(context.Response,upstreamResponse, upstreamResponse.ContentLength / 2, cancellationToken); + Close(context); + return; + case "pa": + // Partial Response (full headers, 50% of body), then abort (TCP RST) + await SendDownstreamResponse(context.Response,upstreamResponse, upstreamResponse.ContentLength / 2, cancellationToken); + Abort(context); + return; + case "pn": + // Partial Response (full headers, 50% of body), then finish normally + await SendDownstreamResponse(context.Response, upstreamResponse, upstreamResponse.ContentLength / 2, cancellationToken); + return; + case "n": + // No response, then wait indefinitely + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + return; + case "nc": + // No response, then close (TCP FIN) + Close(context); + return; + case "na": + // No response, then abort (TCP RST) + Abort(context); + return; + default: + // can't really happen since we validated options before calling into this method. + throw new ArgumentException($"Invalid fault mode: {fault}", nameof(fault)); + } + } + + private async Task SendDownstreamResponse(HttpResponse response, UpstreamResponse upstreamResponse, long contentBytes, CancellationToken cancellationToken) + { + response.StatusCode = upstreamResponse.StatusCode; + foreach (var header in upstreamResponse.Headers) + { + response.Headers.Add(header.Key, header.Value); + } + + _logger.LogInformation("Started writing response body, {actualLength}", contentBytes); + + byte[] buffer = ArrayPool.Shared.Rent(81920); + + try + { + using Stream source = await upstreamResponse.GetContentStreamAsync(cancellationToken); + for (long remaining = contentBytes; remaining > 0;) + { + int read = await source.ReadAsync(buffer, 0, (int)Math.Min(buffer.Length, remaining), cancellationToken); + if (read <= 0) + { + break; + } + + remaining -= read; + await response.Body.WriteAsync(buffer, 0, read, cancellationToken); + } + + await response.Body.FlushAsync(); + } + catch (Exception ex) + { + _logger.LogError(ex, "Can't write response body"); + } + finally + { + // disponse content as early as possible (before infinite wait that might happen later) + // so that underlying connection returns to connection pool + // and we won't run out of them + upstreamResponse.Dispose(); + ArrayPool.Shared.Return(buffer); + _logger.LogInformation("Finished writing response body"); + } + } + + // Close the TCP connection by sending FIN + private void Close(HttpContext context) + { + context.Abort(); + } + + // Abort the TCP connection by sending RST + private void Abort(HttpContext context) + { + // SocketConnection registered "this" as the IConnectionIdFeature among other things. + var socketConnection = context.Features.Get(); + var socket = (Socket)socketConnection.GetType().GetField("_socket", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(socketConnection); + socket.LingerState = new LingerOption(true, 0); + socket.Dispose(); + } + + private bool ValidateOrReadFaultMode(string headerValue, out string fault) + { + fault = headerValue ?? Utils.ReadSelectionFromConsole(); + if (!Utils.FaultModes.TryGetValue(fault, out var description)) + { + _logger.LogError("Unknown {ResponseSelectionHeader} value - {fault}.", Utils.ResponseSelectionHeader, fault); + return false; + } + + _logger.LogInformation("Using response option '{description}' from header value.", description); + return true; + } + } +} diff --git a/tools/http-fault-injector/Azure.Sdk.Tools.HttpFaultInjector/Program.cs b/tools/http-fault-injector/Azure.Sdk.Tools.HttpFaultInjector/Program.cs index a898879ba13..6d1672f5d65 100644 --- a/tools/http-fault-injector/Azure.Sdk.Tools.HttpFaultInjector/Program.cs +++ b/tools/http-fault-injector/Azure.Sdk.Tools.HttpFaultInjector/Program.cs @@ -1,59 +1,18 @@ using CommandLine; using Microsoft.AspNetCore.Builder; -using Microsoft.AspNetCore.Connections.Features; using Microsoft.AspNetCore.Hosting; -using Microsoft.AspNetCore.Http; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Primitives; +using Microsoft.AspNetCore.HttpLogging; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using OpenTelemetry.Logs; +using OpenTelemetry.Resources; using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Net; using System.Net.Http; -using System.Net.Sockets; -using System.Reflection; -using System.Threading; -using System.Threading.Tasks; namespace Azure.Sdk.Tools.HttpFaultInjector { public static class Program { - private static HttpClient _httpClient; - - private static readonly List<(string Option, string Description)> _selectionDescriptions = new List<(string Option, string Description)>() - { - ("f", "Full response"), - ("p", "Partial Response (full headers, 50% of body), then wait indefinitely"), - ("pc", "Partial Response (full headers, 50% of body), then close (TCP FIN)"), - ("pa", "Partial Response (full headers, 50% of body), then abort (TCP RST)"), - ("pn", "Partial Response (full headers, 50% of body), then finish normally"), - ("n", "No response, then wait indefinitely"), - ("nc", "No response, then close (TCP FIN)"), - ("na", "No response, then abort (TCP RST)") - }; - - private static readonly string[] _excludedRequestHeaders = new string[] { - // Only applies to request between client and proxy - "Proxy-Connection", - - // "X-Upstream-Base-Uri" in original request is used as the Base URI in the upstream request - "X-Upstream-Base-Uri", - "Host", - - _responseSelectionHeader - }; - - // Headers which must be set on HttpContent instead of HttpRequestMessage - private static readonly string[] _contentRequestHeaders = new string[] { - "Content-Length", - "Content-Type", - }; - - private const string _responseSelectionHeader = "x-ms-faultinjector-response-option"; - private class Options { [Option('i', "insecure", Default = false, HelpText = "Allow insecure upstream SSL certs")] @@ -71,364 +30,57 @@ public static void Main(string[] args) settings.HelpWriter = Console.Error; }); - parser.ParseArguments(args).WithParsed(options => Run(options)); + parser.ParseArguments(args).WithParsed(options => Run(options, args)); } - private static void Run(Options options) + private static void Run(Options options, string[] args) { TimeSpan keepAlive = TimeSpan.FromSeconds(options.KeepAliveTimeout); - if (options.Insecure) + var builder = WebApplication.CreateBuilder(args); + builder.WebHost.ConfigureKestrel(kestrelOptions => { - _httpClient = new HttpClient(new HttpClientHandler() - { - // Allow insecure SSL certs - ServerCertificateCustomValidationCallback = (message, cert, chain, errors) => true - }); - } - else + kestrelOptions.Limits.KeepAliveTimeout = keepAlive; + }); + + builder.Services.AddHttpLogging(logging => { - _httpClient = new HttpClient(); - } + logging.LoggingFields = HttpLoggingFields.RequestPropertiesAndHeaders | HttpLoggingFields.ResponsePropertiesAndHeaders; + logging.RequestHeaders.Add(Utils.ResponseSelectionHeader); + logging.RequestHeaders.Add(Utils.UpstreamBaseUriHeader); + logging.RequestHeaders.Add("ETag"); + logging.ResponseHeaders.Add("ETag"); + }); // TODO: we can switch to SocketsHttpHandler and configure read/write/connect timeouts separately // for now let's just set upstream timeout to be slightly bigger than client timeout. - _httpClient.Timeout = keepAlive + TimeSpan.FromSeconds(1); - new WebHostBuilder() - .UseKestrel(kestrelOptions => - { - kestrelOptions.Listen(IPAddress.Any, 7777); - kestrelOptions.Listen(IPAddress.Any, 7778, listenOptions => - { - listenOptions.UseHttps(); - }); - kestrelOptions.Limits.KeepAliveTimeout = keepAlive; - }) - .Configure(app => app.Run(async context => - { - try - { - using var upstreamResponse = await SendUpstreamRequest(context.Request, context.RequestAborted); - - // Attempt to remove the response selection header and use its value to handle the response selection. - if (context.Request.Headers.Remove(_responseSelectionHeader, out var selection)) - { - string optionDescription = _selectionDescriptions.FirstOrDefault(kvp => kvp.Option.Equals(selection)).Description; - if (string.IsNullOrEmpty(optionDescription)) - { - Console.WriteLine($"Invalid {_responseSelectionHeader} value {selection}."); - } - else if (await TryHandleResponseOption(selection, context, upstreamResponse)) - { - Console.WriteLine($"Using response option {optionDescription} from header value."); - return; - } - } - - // If we were passed an invalid response selection header, or none, continue prompting the user for input and attempt to handle the response. - while (true) - { - Console.WriteLine(); - - Console.WriteLine("Select a response then press ENTER:"); - foreach (var selectionDescription in _selectionDescriptions) - { - Console.WriteLine($"{selectionDescription.Option}: {selectionDescription.Description}"); - } - - Console.WriteLine(); - - selection = Console.ReadLine(); - - if (await TryHandleResponseOption(selection, context, upstreamResponse)) - { - return; - } - } - } - catch (Exception e) - { - Console.WriteLine(e); - throw; - } - })) - .Build() - .Run(); - } - - private static async Task SendUpstreamRequest(HttpRequest request, CancellationToken cancellationToken) - { - Console.WriteLine(); - Log("Incoming Request"); - - var incomingUriBuilder = new UriBuilder() { - Scheme = request.Scheme, - Host = request.Host.Host, - Path = request.Path.Value, - Query = request.QueryString.Value, - }; - if (request.Host.Port.HasValue) - { - incomingUriBuilder.Port = request.Host.Port.Value; - } - var incomingUri = incomingUriBuilder.Uri; - - Log($"URL: {incomingUri}"); - - Log("Headers:"); - foreach (var header in request.Headers) - { - Log($" {header.Key}:{header.Value}"); - } - - var upstreamUriBuilder = new UriBuilder(request.Headers["X-Upstream-Base-Uri"]) - { - Path = request.Path.Value, - Query = request.QueryString.Value, - }; - - var upstreamUri = upstreamUriBuilder.Uri; - - Console.WriteLine(); - Log("Upstream Request"); - Log($"URL: {upstreamUri}"); - - using var upstreamRequest = new HttpRequestMessage(new HttpMethod(request.Method), upstreamUri); - Log("Headers:"); - - if (request.ContentLength > 0) - { - upstreamRequest.Content = new StreamContent(request.Body); - - foreach (var header in request.Headers.Where(h => _contentRequestHeaders.Contains(h.Key))) - { - Log($" {header.Key}:{header.Value.First()}"); - upstreamRequest.Content.Headers.Add(header.Key, values: header.Value); - } - } + var httpClientBuilder = builder.Services.AddHttpClient("upstream", client => client.Timeout = keepAlive + TimeSpan.FromSeconds(1)); - foreach (var header in request.Headers.Where(h => !_excludedRequestHeaders.Contains(h.Key) && !_contentRequestHeaders.Contains(h.Key))) - { - Log($" {header.Key}:{header.Value.First()}"); - if (!upstreamRequest.Headers.TryAddWithoutValidation(header.Key, values: header.Value)) - { - throw new InvalidOperationException($"Could not add header {header.Key} with value {header.Value}"); - } - } - - Log("Sending request to upstream server..."); - var upstreamResponseMessage = await _httpClient.SendAsync(upstreamRequest, HttpCompletionOption.ResponseHeadersRead, cancellationToken); - Console.WriteLine(); - Log("Upstream Response"); - var headers = new List>(); - - Log($"StatusCode: {upstreamResponseMessage.StatusCode}"); - - Log("Headers:"); - - foreach (var header in upstreamResponseMessage.Headers) - { - Log($" {header.Key}:{header.Value.First()}"); - - // Must skip "Transfer-Encoding" header, since if it's set manually Kestrel requires you to implement - // your own chunking. - if (string.Equals(header.Key, "Transfer-Encoding", StringComparison.OrdinalIgnoreCase)) - { - continue; - } - - headers.Add(new KeyValuePair(header.Key, header.Value.ToArray())); - } - - foreach (var header in upstreamResponseMessage.Content.Headers) - { - Log($" {header.Key}:{header.Value.First()}"); - headers.Add(new KeyValuePair(header.Key, header.Value.ToArray())); - } - - Log($"ContentLength: {upstreamResponseMessage.Content.Headers.ContentLength}, is chunked: {upstreamResponseMessage.Headers.TransferEncodingChunked}"); - - var response = await UpstreamResponse.FromContent(upstreamResponseMessage.Content, cancellationToken); - response.StatusCode = (int)upstreamResponseMessage.StatusCode; - response.Headers = headers.ToArray(); - - return response; - } - - private static async Task TryHandleResponseOption(string selection, HttpContext context, UpstreamResponse upstreamResponse) - { - switch (selection) - { - case "f": - // Full response - await SendDownstreamResponse(upstreamResponse, context.Response, upstreamResponse.ContentLength, context.RequestAborted); - return true; - case "p": - // Partial Response (full headers, 50% of body), then wait indefinitely - await SendDownstreamResponse(upstreamResponse, context.Response, upstreamResponse.ContentLength / 2, context.RequestAborted); - await Task.Delay(Timeout.InfiniteTimeSpan, context.RequestAborted); - return true; - case "pc": - // Partial Response (full headers, 50% of body), then close (TCP FIN) - await SendDownstreamResponse(upstreamResponse, context.Response, upstreamResponse.ContentLength / 2, context.RequestAborted); - Close(context); - return true; - case "pa": - // Partial Response (full headers, 50% of body), then abort (TCP RST) - await SendDownstreamResponse(upstreamResponse, context.Response, upstreamResponse.ContentLength / 2, context.RequestAborted); - Abort(context); - return true; - case "pn": - // Partial Response (full headers, 50% of body), then finish normally - await SendDownstreamResponse(upstreamResponse, context.Response, upstreamResponse.ContentLength / 2, context.RequestAborted); - return true; - case "n": - // No response, then wait indefinitely - await Task.Delay(Timeout.InfiniteTimeSpan, context.RequestAborted); - return true; - case "nc": - // No response, then close (TCP FIN) - Close(context); - return true; - case "na": - // No response, then abort (TCP RST) - Abort(context); - return true; - default: - Console.WriteLine($"Invalid selection: {selection}"); - return false; - } - } - - private static async Task SendDownstreamResponse(UpstreamResponse upstreamResponse, HttpResponse response, long? contentBytes, CancellationToken cancellationToken) - { - Console.WriteLine(); - - Log("Sending downstream response..."); - - response.StatusCode = upstreamResponse.StatusCode; - - Log($"StatusCode: {upstreamResponse.StatusCode}"); - - Log("Headers:"); - foreach (var header in upstreamResponse.Headers) - { - Log($" {header.Key}:{header.Value}"); - response.Headers.Add(header.Key, header.Value); - } - - long count = contentBytes ?? long.MaxValue; - - Log($"Writing response body of {count} bytes..."); - - using Stream source = await upstreamResponse.GetContentStreamAsync(cancellationToken); - byte[] buffer = new byte[8192]; - - try + if (options.Insecure) { - for (long remaining = count; remaining > 0;) + httpClientBuilder.ConfigurePrimaryHttpMessageHandler(() => { - int read = await source.ReadAsync(buffer, 0, (int)Math.Min(buffer.Length, remaining), cancellationToken); - if (read <= 0) + return new HttpClientHandler() { - break; - } - - remaining -= read; - await response.Body.WriteAsync(buffer, 0, read, cancellationToken); - } - - await response.Body.FlushAsync(); - Log($"Finished writing response body"); - } - catch (Exception ex) - { - Log(ex.ToString()); - } - finally - { - // disponse content as early as possible (before infinite wait that might happen later) - // so that underlying connection returns to connection pool - // and we won't run out of them - upstreamResponse.Dispose(); - } - } - - // Close the TCP connection by sending FIN - private static void Close(HttpContext context) - { - context.Abort(); - } - - // Abort the TCP connection by sending RST - private static void Abort(HttpContext context) - { - // SocketConnection registered "this" as the IConnectionIdFeature among other things. - var socketConnection = context.Features.Get(); - var socket = (Socket)socketConnection.GetType().GetField("_socket", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(socketConnection); - socket.LingerState = new LingerOption(true, 0); - socket.Dispose(); - } - - private static void Log(object value) - { - Console.WriteLine($"[{DateTime.Now:hh:mm:ss.fff}] {value}"); - } - - private class UpstreamResponse : IDisposable - { - private readonly HttpContent _content = null; - private readonly Stream _contentStream = null; - private UpstreamResponse(HttpContent content) - { - _content = content; - ContentLength = _content.Headers.ContentLength ?? throw new ArgumentNullException("ContentLength must not be null."); - } - - private UpstreamResponse(MemoryStream contentStream) - { - _contentStream = contentStream; - ContentLength = contentStream.Length; - } - - public int StatusCode { get; set; } - public KeyValuePair[] Headers { get; set; } - public async Task GetContentStreamAsync(CancellationToken cancellationToken) - { - return _contentStream != null ? _contentStream : await _content.ReadAsStreamAsync(cancellationToken); - } - - public long ContentLength { get; } - - public void Dispose() - { - _content?.Dispose(); - _contentStream?.Dispose(); - } - - public static async Task FromContent(HttpContent content, CancellationToken cancellationToken) - { - if (content.Headers.ContentLength == null) - { - MemoryStream contentStream = await BufferContentAsync(content, cancellationToken); - // we no longer need that content and can let the connection go back to the pool. - content.Dispose(); - return new UpstreamResponse(contentStream); - } - - return new UpstreamResponse(content); + // Allow insecure SSL certs + ServerCertificateCustomValidationCallback = (message, cert, chain, errors) => true + }; + }); } - private static async Task BufferContentAsync(HttpContent content, CancellationToken cancellationToken) + builder.Logging.ClearProviders(); + builder.Logging.AddOpenTelemetry(o => { - Debug.Assert(content.Headers.ContentLength == null, "We should not buffer content if length is available."); - - Log("Response does not have content length (is chunked or malformed) and is being buffered"); - byte[] contentBytes = await content.ReadAsByteArrayAsync(cancellationToken); - Log($"Content was buffered, total length - {contentBytes.Length}."); - - // comparing to buffering full content memory stream allocation is not such a big deal. - return new MemoryStream(contentBytes); - } + o.SetResourceBuilder(ResourceBuilder.CreateEmpty()); + o.IncludeFormattedMessage = false; + o.IncludeScopes = false; + o.ParseStateValues = true; + // can add more exporters, e.g. ApplicationInsights + o.AddConsoleExporter(); + }); + var app = builder.Build(); + app.UseHttpLogging(); + app.UseMiddleware(); + app.Run(); } } } diff --git a/tools/http-fault-injector/Azure.Sdk.Tools.HttpFaultInjector/UpstreamResponse.cs b/tools/http-fault-injector/Azure.Sdk.Tools.HttpFaultInjector/UpstreamResponse.cs new file mode 100644 index 00000000000..fdb9477e9c0 --- /dev/null +++ b/tools/http-fault-injector/Azure.Sdk.Tools.HttpFaultInjector/UpstreamResponse.cs @@ -0,0 +1,43 @@ +using Microsoft.Extensions.Primitives; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Net.Http; +using System.Threading.Tasks; +using System.Threading; + +namespace Azure.Sdk.Tools.HttpFaultInjector +{ + public class UpstreamResponse : IDisposable + { + private readonly HttpContent _content = null; + private readonly Stream _contentStream = null; + public UpstreamResponse(HttpContent content) + { + _content = content; + ContentLength = _content.Headers.ContentLength ?? throw new ArgumentNullException("ContentLength must not be null."); + } + + public UpstreamResponse(MemoryStream contentStream) + { + _contentStream = contentStream; + ContentLength = contentStream.Length; + } + + public int StatusCode { get; set; } + public IEnumerable> Headers { get; set; } + public async Task GetContentStreamAsync(CancellationToken cancellationToken) + { + return _contentStream != null ? _contentStream : await _content.ReadAsStreamAsync(cancellationToken); + } + + public long ContentLength { get; } + + public void Dispose() + { + _content?.Dispose(); + _contentStream?.Dispose(); + } + } +} diff --git a/tools/http-fault-injector/Azure.Sdk.Tools.HttpFaultInjector/Utils.cs b/tools/http-fault-injector/Azure.Sdk.Tools.HttpFaultInjector/Utils.cs new file mode 100644 index 00000000000..a53e05720b7 --- /dev/null +++ b/tools/http-fault-injector/Azure.Sdk.Tools.HttpFaultInjector/Utils.cs @@ -0,0 +1,61 @@ +using System; +using System.Collections.Generic; + +namespace Azure.Sdk.Tools.HttpFaultInjector +{ + public static class Utils + { + public static readonly IDictionary FaultModes = new Dictionary() + { + { "f", "Full response" }, + { "p", "Partial Response (full headers, 50% of body), then wait indefinitely" }, + {"pc", "Partial Response (full headers, 50% of body), then close (TCP FIN)" }, + {"pa", "Partial Response (full headers, 50% of body), then abort (TCP RST)" }, + {"pn", "Partial Response (full headers, 50% of body), then finish normally" }, + {"n", "No response, then wait indefinitely"}, + {"nc", "No response, then close (TCP FIN)" }, + {"na", "No response, then abort (TCP RST)" } + }; + + public static readonly string[] ExcludedRequestHeaders = new string[] { + // Only applies to request between client and proxy + "Proxy-Connection", + + // "X-Upstream-Base-Uri" in original request is used as the Base URI in the upstream request + UpstreamBaseUriHeader, + "Host", + + ResponseSelectionHeader + }; + + // Headers which must be set on HttpContent instead of HttpRequestMessage + public static readonly string[] ContentRequestHeaders = new string[] { + "Content-Length", + "Content-Type", + }; + + public const string ResponseSelectionHeader = "x-ms-faultinjector-response-option"; + public const string UpstreamBaseUriHeader = "X-Upstream-Base-Uri"; + + public static string ReadSelectionFromConsole() + { + string fault; + do + { + Console.WriteLine(); + + Console.WriteLine("Select a response fault mode then press ENTER:"); + foreach (var kvp in FaultModes) + { + Console.WriteLine($"{kvp.Key}: {kvp.Value}"); + } + + Console.WriteLine(); + + fault = Console.ReadLine(); + } while (fault == null || !FaultModes.ContainsKey(fault)); + + return fault; + } + } +} diff --git a/tools/http-fault-injector/Azure.Sdk.Tools.HttpFaultInjector/appsettings.json b/tools/http-fault-injector/Azure.Sdk.Tools.HttpFaultInjector/appsettings.json new file mode 100644 index 00000000000..dde8368257d --- /dev/null +++ b/tools/http-fault-injector/Azure.Sdk.Tools.HttpFaultInjector/appsettings.json @@ -0,0 +1,11 @@ +{ + "Logging": { + "LogLevel": { + "Azure": "Information", + "Default": "Warning", + "System.Net.Http.HttpClient.upstream": "Trace", + "Microsoft.AspNetCore.HttpLogging.HttpLoggingMiddleware": "Information" + } + }, + "Urls": "http://+:7777/;https://+:7778/" +}