From c7c8dbdfbea6f3e2815f918344419d19f1d3ad83 Mon Sep 17 00:00:00 2001 From: martintmk <103487740+martintmk@users.noreply.github.com> Date: Sun, 23 Apr 2023 15:36:31 +0200 Subject: [PATCH] Migrate `CircuitStateController` and implement `CircutBreakerResilienceStrategy ` (#1152) --- build.cake | 1 + src/Directory.Packages.props | 2 +- .../AdvancedCircuitBreakerOptionsTests.cs | 4 +- .../CircuitBreakerManualControlTests.cs | 23 +- .../CircuitBreakerOptionsTests.cs | 4 +- ...itBreakerResilienceStrategyBuilderTests.cs | 83 ++++ .../CircuitBreakerResilienceStrategyTests.cs | 105 ++++- .../CircuitBreakerStateProviderTests.cs | 11 +- .../AdvancedCircuitBehaviorTests.cs | 22 + .../Controller/CircuitStateControllerTests.cs | 418 ++++++++++++++++++ ...ConsecutiveFailuresCircuitBehaviorTests.cs | 49 ++ .../Controller/ScheduledTaskExecutorTests.cs | 149 +++++++ .../OnCircuitClosedArgumentsTests.cs | 3 +- .../OnCircuitOpenedArgumentsTests.cs | 3 +- ...seCircuitBreakerStrategyOptions.TResult.cs | 30 ++ .../BaseCircuitBreakerStrategyOptions.cs | 30 ++ .../CircuitBreaker/BrokenCircuitException.cs | 3 + .../CircuitBreaker/CircuitBreakerConstants.cs | 6 +- .../CircuitBreakerManualControl.cs | 19 +- .../CircuitBreakerResilienceStrategy.cs | 69 ++- ...akerResilienceStrategyBuilderExtensions.cs | 28 +- .../CircuitBreakerStateProvider.cs | 15 +- .../Controller/AdvancedCircuitBehavior.cs | 18 + .../Controller/CircuitBehavior.cs | 13 + .../Controller/CircuitStateController.cs | 326 ++++++++++++++ .../ConsecutiveFailuresCircuitBehavior.cs | 37 ++ .../Controller/ScheduledTaskExecutor.cs | 87 ++++ .../OnCircuitClosedArguments.cs | 13 +- .../OnCircuitOpenedArguments.cs | 8 +- src/Polly.TestUtils/TestUtilities.cs | 14 + 30 files changed, 1537 insertions(+), 56 deletions(-) create mode 100644 src/Polly.Core.Tests/CircuitBreaker/Controller/AdvancedCircuitBehaviorTests.cs create mode 100644 src/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs create mode 100644 src/Polly.Core.Tests/CircuitBreaker/Controller/ConsecutiveFailuresCircuitBehaviorTests.cs create mode 100644 src/Polly.Core.Tests/CircuitBreaker/Controller/ScheduledTaskExecutorTests.cs create mode 100644 src/Polly.Core/CircuitBreaker/Controller/AdvancedCircuitBehavior.cs create mode 100644 src/Polly.Core/CircuitBreaker/Controller/CircuitBehavior.cs create mode 100644 src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs create mode 100644 src/Polly.Core/CircuitBreaker/Controller/ConsecutiveFailuresCircuitBehavior.cs create mode 100644 src/Polly.Core/CircuitBreaker/Controller/ScheduledTaskExecutor.cs diff --git a/build.cake b/build.cake index f18746611d9..bcc65bc93ca 100644 --- a/build.cake +++ b/build.cake @@ -211,6 +211,7 @@ Task("__RunTests") Configuration = configuration, Loggers = loggers, NoBuild = true, + ArgumentCustomization = args => args.Append($"--blame-hang-timeout 10s") }); } }); diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index 598d85c4e83..2294520024f 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -8,7 +8,7 @@ - + diff --git a/src/Polly.Core.Tests/CircuitBreaker/AdvancedCircuitBreakerOptionsTests.cs b/src/Polly.Core.Tests/CircuitBreaker/AdvancedCircuitBreakerOptionsTests.cs index 5bb613e6b7b..6ccadeaa77a 100644 --- a/src/Polly.Core.Tests/CircuitBreaker/AdvancedCircuitBreakerOptionsTests.cs +++ b/src/Polly.Core.Tests/CircuitBreaker/AdvancedCircuitBreakerOptionsTests.cs @@ -97,10 +97,10 @@ public async Task AsNonGenericOptions_Ok() (await converted.ShouldHandle.CreateHandler()!.ShouldHandleAsync(new Outcome(new InvalidOperationException()), new CircuitBreakerPredicateArguments(context))).Should().BeTrue(); - await converted.OnClosed.CreateHandler()!.HandleAsync(new Outcome(new InvalidOperationException()), new OnCircuitClosedArguments(context)); + await converted.OnClosed.CreateHandler()!.HandleAsync(new Outcome(new InvalidOperationException()), new OnCircuitClosedArguments(context, true)); onResetCalled.Should().BeTrue(); - await converted.OnOpened.CreateHandler()!.HandleAsync(new Outcome(new InvalidOperationException()), new OnCircuitOpenedArguments(context, TimeSpan.Zero)); + await converted.OnOpened.CreateHandler()!.HandleAsync(new Outcome(new InvalidOperationException()), new OnCircuitOpenedArguments(context, TimeSpan.Zero, true)); onBreakCalled.Should().BeTrue(); await converted.OnHalfOpened.CreateHandler()!(new OnCircuitHalfOpenedArguments(context)); diff --git a/src/Polly.Core.Tests/CircuitBreaker/CircuitBreakerManualControlTests.cs b/src/Polly.Core.Tests/CircuitBreaker/CircuitBreakerManualControlTests.cs index a16ceebf55a..16f5f858ee2 100644 --- a/src/Polly.Core.Tests/CircuitBreaker/CircuitBreakerManualControlTests.cs +++ b/src/Polly.Core.Tests/CircuitBreaker/CircuitBreakerManualControlTests.cs @@ -8,7 +8,7 @@ public class CircuitBreakerManualControlTests [Fact] public void Ctor_Ok() { - var control = new CircuitBreakerManualControl(); + using var control = new CircuitBreakerManualControl(); control.IsInitialized.Should().BeFalse(); } @@ -16,7 +16,7 @@ public void Ctor_Ok() [Fact] public async Task IsolateAsync_NotInitialized_Throws() { - var control = new CircuitBreakerManualControl(); + using var control = new CircuitBreakerManualControl(); await control .Invoking(c => c.IsolateAsync(CancellationToken.None)) @@ -27,10 +27,10 @@ await control [Fact] public async Task ResetAsync_NotInitialized_Throws() { - var control = new CircuitBreakerManualControl(); + using var control = new CircuitBreakerManualControl(); await control - .Invoking(c => c.ResetAsync(CancellationToken.None)) + .Invoking(c => c.CloseAsync(CancellationToken.None)) .Should() .ThrowAsync(); } @@ -38,11 +38,11 @@ await control [Fact] public void Initialize_Twice_Throws() { - var control = new CircuitBreakerManualControl(); - control.Initialize(_ => Task.CompletedTask, _ => Task.CompletedTask); + using var control = new CircuitBreakerManualControl(); + control.Initialize(_ => Task.CompletedTask, _ => Task.CompletedTask, () => { }); control - .Invoking(c => c.Initialize(_ => Task.CompletedTask, _ => Task.CompletedTask)) + .Invoking(c => c.Initialize(_ => Task.CompletedTask, _ => Task.CompletedTask, () => { })) .Should() .Throw(); } @@ -53,6 +53,7 @@ public async Task Initialize_Ok() var control = new CircuitBreakerManualControl(); var isolateCalled = false; var resetCalled = false; + var disposeCalled = false; control.Initialize( context => @@ -68,12 +69,16 @@ public async Task Initialize_Ok() context.IsSynchronous.Should().BeFalse(); resetCalled = true; return Task.CompletedTask; - }); + }, + () => disposeCalled = true); await control.IsolateAsync(CancellationToken.None); - await control.ResetAsync(CancellationToken.None); + await control.CloseAsync(CancellationToken.None); + + control.Dispose(); isolateCalled.Should().BeTrue(); resetCalled.Should().BeTrue(); + disposeCalled.Should().BeTrue(); } } diff --git a/src/Polly.Core.Tests/CircuitBreaker/CircuitBreakerOptionsTests.cs b/src/Polly.Core.Tests/CircuitBreaker/CircuitBreakerOptionsTests.cs index d1d20b43782..cc15d1f7ae8 100644 --- a/src/Polly.Core.Tests/CircuitBreaker/CircuitBreakerOptionsTests.cs +++ b/src/Polly.Core.Tests/CircuitBreaker/CircuitBreakerOptionsTests.cs @@ -85,10 +85,10 @@ public async Task AsNonGenericOptions_Ok() (await converted.ShouldHandle.CreateHandler()!.ShouldHandleAsync(new Outcome(new InvalidOperationException()), new CircuitBreakerPredicateArguments(context))).Should().BeTrue(); - await converted.OnClosed.CreateHandler()!.HandleAsync(new Outcome(new InvalidOperationException()), new OnCircuitClosedArguments(context)); + await converted.OnClosed.CreateHandler()!.HandleAsync(new Outcome(new InvalidOperationException()), new OnCircuitClosedArguments(context, true)); onResetCalled.Should().BeTrue(); - await converted.OnOpened.CreateHandler()!.HandleAsync(new Outcome(new InvalidOperationException()), new OnCircuitOpenedArguments(context, TimeSpan.Zero)); + await converted.OnOpened.CreateHandler()!.HandleAsync(new Outcome(new InvalidOperationException()), new OnCircuitOpenedArguments(context, TimeSpan.Zero, true)); onBreakCalled.Should().BeTrue(); await converted.OnHalfOpened.CreateHandler()!(new OnCircuitHalfOpenedArguments(context)); diff --git a/src/Polly.Core.Tests/CircuitBreaker/CircuitBreakerResilienceStrategyBuilderTests.cs b/src/Polly.Core.Tests/CircuitBreaker/CircuitBreakerResilienceStrategyBuilderTests.cs index df05852315d..c31f8ad75ab 100644 --- a/src/Polly.Core.Tests/CircuitBreaker/CircuitBreakerResilienceStrategyBuilderTests.cs +++ b/src/Polly.Core.Tests/CircuitBreaker/CircuitBreakerResilienceStrategyBuilderTests.cs @@ -74,4 +74,87 @@ public void AddAdvancedCircuitBreaker_Validation() .Throw() .WithMessage("The advanced circuit breaker strategy options are invalid.*"); } + + [Fact] + public void AddCircuitBreaker_IntegrationTest() + { + int opened = 0; + int closed = 0; + int halfOpened = 0; + + var options = new CircuitBreakerStrategyOptions + { + FailureThreshold = 5, + BreakDuration = TimeSpan.FromMilliseconds(500), + }; + + options.ShouldHandle.HandleResult(-1); + options.OnOpened.Register(() => opened++); + options.OnClosed.Register(() => closed++); + options.OnHalfOpened.Register(() => halfOpened++); + + var timeProvider = new FakeTimeProvider(); + var strategy = new ResilienceStrategyBuilder { TimeProvider = timeProvider.Object }.AddCircuitBreaker(options).Build(); + var time = DateTime.UtcNow; + timeProvider.Setup(v => v.UtcNow).Returns(() => time); + + for (int i = 0; i < options.FailureThreshold; i++) + { + strategy.Execute(_ => -1); + } + + // Circuit opened + opened.Should().Be(1); + halfOpened.Should().Be(0); + closed.Should().Be(0); + Assert.Throws>(() => strategy.Execute(_ => 0)); + + // Circuit Half Opened + time += options.BreakDuration; + strategy.Execute(_ => -1); + Assert.Throws>(() => strategy.Execute(_ => 0)); + opened.Should().Be(2); + halfOpened.Should().Be(1); + closed.Should().Be(0); + + // Now close it + time += options.BreakDuration; + strategy.Execute(_ => 0); + opened.Should().Be(2); + halfOpened.Should().Be(2); + closed.Should().Be(1); + } + + [Fact] + public void AddAdvancedCircuitBreaker_IntegrationTest() + { + var options = new AdvancedCircuitBreakerStrategyOptions + { + BreakDuration = TimeSpan.FromMilliseconds(500), + }; + + options.ShouldHandle.HandleResult(-1); + options.OnOpened.Register(() => { }); + options.OnClosed.Register(() => { }); + options.OnHalfOpened.Register(() => { }); + + var timeProvider = new FakeTimeProvider(); + var strategy = new ResilienceStrategyBuilder { TimeProvider = timeProvider.Object }.AddAdvancedCircuitBreaker(options).Build(); + var time = DateTime.UtcNow; + timeProvider.Setup(v => v.UtcNow).Returns(() => time); + + strategy.Should().BeOfType(); + } + + [Fact] + public void AddCircuitBreaker_UnrecognizedOptions_Throws() + { + var builder = new ResilienceStrategyBuilder(); + + builder.Invoking(b => b.AddCircuitBreakerCore(new DummyOptions()).Build()).Should().Throw(); + } + + private class DummyOptions : BaseCircuitBreakerStrategyOptions + { + } } diff --git a/src/Polly.Core.Tests/CircuitBreaker/CircuitBreakerResilienceStrategyTests.cs b/src/Polly.Core.Tests/CircuitBreaker/CircuitBreakerResilienceStrategyTests.cs index a95f72e4c63..eb841b5f22a 100644 --- a/src/Polly.Core.Tests/CircuitBreaker/CircuitBreakerResilienceStrategyTests.cs +++ b/src/Polly.Core.Tests/CircuitBreaker/CircuitBreakerResilienceStrategyTests.cs @@ -4,28 +4,127 @@ namespace Polly.Core.Tests.CircuitBreaker; -public class CircuitBreakerResilienceStrategyTests +public class CircuitBreakerResilienceStrategyTests : IDisposable { private readonly FakeTimeProvider _timeProvider; + private readonly Mock _behavior; private readonly ResilienceStrategyTelemetry _telemetry; + private readonly CircuitBreakerStrategyOptions _options; + private readonly CircuitStateController _controller; public CircuitBreakerResilienceStrategyTests() { _timeProvider = new FakeTimeProvider(); + _timeProvider.Setup(v => v.UtcNow).Returns(DateTime.UtcNow); + _behavior = new Mock(MockBehavior.Strict); _telemetry = TestUtilities.CreateResilienceTelemetry(Mock.Of()); + _options = new CircuitBreakerStrategyOptions(); + _controller = new CircuitStateController( + new CircuitBreakerStrategyOptions(), + _behavior.Object, + _timeProvider.Object, + _telemetry); } [Fact] public void Ctor_Ok() { - Create().Should().NotBeNull(); + this.Invoking(_ => Create()).Should().NotThrow(); } + [Fact] + public void Ctor_StateProvider_EnsureAttached() + { + _options.StateProvider = new CircuitBreakerStateProvider(); + Create(); + + _options.StateProvider.IsInitialized.Should().BeTrue(); + + _options.StateProvider.CircuitState.Should().Be(CircuitState.Closed); + _options.StateProvider.LastHandledOutcome.Should().Be(null); + } + + [Fact] + public async Task Ctor_ManualControl_EnsureAttached() + { + _options.ShouldHandle.HandleException(); + _options.ManualControl = new CircuitBreakerManualControl(); + var strategy = Create(); + + _options.ManualControl.IsInitialized.Should().BeTrue(); + + await _options.ManualControl.IsolateAsync(CancellationToken.None); + strategy.Invoking(s => s.Execute(_ => { })).Should().Throw(); + + _behavior.Setup(v => v.OnCircuitClosed()); + await _options.ManualControl.CloseAsync(CancellationToken.None); + + _behavior.Setup(v => v.OnActionSuccess(CircuitState.Closed)); + strategy.Invoking(s => s.Execute(_ => { })).Should().NotThrow(); + + _options.ManualControl.Dispose(); + strategy.Invoking(s => s.Execute(_ => { })).Should().Throw(); + + _behavior.VerifyAll(); + } + + [Fact] + public void Execute_HandledResult_OnFailureCalled() + { + _options.ShouldHandle.HandleResult(-1); + var strategy = Create(); + var shouldBreak = false; + + _behavior.Setup(v => v.OnActionFailure(CircuitState.Closed, out shouldBreak)); + strategy.Execute(_ => -1).Should().Be(-1); + + _behavior.VerifyAll(); + } + + [Fact] + public void Execute_UnhandledResult_OnActionSuccess() + { + _options.ShouldHandle.HandleResult(-1); + var strategy = Create(); + + _behavior.Setup(v => v.OnActionSuccess(CircuitState.Closed)); + strategy.Execute(_ => 0).Should().Be(0); + + _behavior.VerifyAll(); + } + + [Fact] + public void Execute_HandledException_OnFailureCalled() + { + _options.ShouldHandle.HandleException(); + var strategy = Create(); + var shouldBreak = false; + + _behavior.Setup(v => v.OnActionFailure(CircuitState.Closed, out shouldBreak)); + + strategy.Invoking(s => s.Execute(_ => throw new InvalidOperationException())).Should().Throw(); + + _behavior.VerifyAll(); + } + + [Fact] + public void Execute_UnhandledException_NoCalls() + { + _options.ShouldHandle.HandleException(); + var strategy = Create(); + + strategy.Invoking(s => s.Execute(_ => throw new ArgumentException())).Should().Throw(); + + _behavior.VerifyNoOtherCalls(); + } + + public void Dispose() => _controller.Dispose(); + [Fact] public void Execute_Ok() { Create().Invoking(s => s.Execute(_ => { })).Should().NotThrow(); } - private CircuitBreakerResilienceStrategy Create() => new(_timeProvider.Object, _telemetry, new CircuitBreakerStrategyOptions()); + private CircuitBreakerResilienceStrategy Create() => new(_options, _controller); } diff --git a/src/Polly.Core.Tests/CircuitBreaker/CircuitBreakerStateProviderTests.cs b/src/Polly.Core.Tests/CircuitBreaker/CircuitBreakerStateProviderTests.cs index 79fba82d76a..d5a4df097d7 100644 --- a/src/Polly.Core.Tests/CircuitBreaker/CircuitBreakerStateProviderTests.cs +++ b/src/Polly.Core.Tests/CircuitBreaker/CircuitBreakerStateProviderTests.cs @@ -1,5 +1,6 @@ using System; using Polly.CircuitBreaker; +using Polly.Strategy; namespace Polly.Core.Tests.CircuitBreaker; @@ -19,16 +20,16 @@ public void NotInitialized_EnsureDefaults() var provider = new CircuitBreakerStateProvider(); provider.CircuitState.Should().Be(CircuitState.Closed); - provider.LastException.Should().Be(null); + provider.LastHandledOutcome.Should().Be(null); } [Fact] public async Task ResetAsync_NotInitialized_Throws() { - var control = new CircuitBreakerManualControl(); + using var control = new CircuitBreakerManualControl(); await control - .Invoking(c => c.ResetAsync(CancellationToken.None)) + .Invoking(c => c.CloseAsync(CancellationToken.None)) .Should() .ThrowAsync(); } @@ -61,11 +62,11 @@ public void Initialize_Ok() () => { exceptionCalled = true; - return new InvalidOperationException(); + return new Outcome(typeof(string), new InvalidOperationException()); }); provider.CircuitState.Should().Be(CircuitState.HalfOpen); - provider.LastException.Should().BeOfType(); + provider.LastHandledOutcome!.Value.Exception.Should().BeOfType(); stateCalled.Should().BeTrue(); exceptionCalled.Should().BeTrue(); diff --git a/src/Polly.Core.Tests/CircuitBreaker/Controller/AdvancedCircuitBehaviorTests.cs b/src/Polly.Core.Tests/CircuitBreaker/Controller/AdvancedCircuitBehaviorTests.cs new file mode 100644 index 00000000000..21ecc5c1a48 --- /dev/null +++ b/src/Polly.Core.Tests/CircuitBreaker/Controller/AdvancedCircuitBehaviorTests.cs @@ -0,0 +1,22 @@ +using Polly.CircuitBreaker; + +namespace Polly.Core.Tests.CircuitBreaker.Controller; +public class AdvancedCircuitBehaviorTests +{ + [Fact] + public void HappyPath() + { + var behavior = new AdvancedCircuitBehavior(); + + behavior + .Invoking(b => + { + behavior.OnActionFailure(CircuitState.Closed, out var shouldBreak); + shouldBreak.Should().BeFalse(); + behavior.OnCircuitClosed(); + behavior.OnActionSuccess(CircuitState.Closed); + }) + .Should() + .NotThrow(); + } +} diff --git a/src/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs b/src/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs new file mode 100644 index 00000000000..68a01caa0cf --- /dev/null +++ b/src/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs @@ -0,0 +1,418 @@ +using System; +using System.Threading.Tasks; +using Moq; +using Polly.CircuitBreaker; +using Polly.Strategy; + +namespace Polly.Core.Tests.CircuitBreaker.Controller; +public class CircuitStateControllerTests +{ + private readonly FakeTimeProvider _timeProvider = new(); + private readonly BaseCircuitBreakerStrategyOptions _options = new CircuitBreakerStrategyOptions(); + private readonly Mock _circuitBehavior = new(MockBehavior.Strict); + private readonly Action _onTelemetry = _ => { }; + private DateTimeOffset _utcNow = DateTimeOffset.UtcNow; + + public CircuitStateControllerTests() => _timeProvider.Setup(v => v.UtcNow).Returns(() => _utcNow); + + [Fact] + public void Ctor_EnsureDefaults() + { + using var controller = CreateController(); + + controller.CircuitState.Should().Be(CircuitState.Closed); + controller.LastException.Should().BeNull(); + controller.LastHandledOutcome.Should().BeNull(); + } + + [Fact] + public async Task IsolateAsync_Ok() + { + // arrange + bool called = false; + _options.OnOpened.Register((outcome, args) => + { + args.BreakDuration.Should().Be(TimeSpan.MaxValue); + args.Context.IsSynchronous.Should().BeFalse(); + args.Context.IsVoid.Should().BeTrue(); + args.IsManual.Should().BeTrue(); + outcome.IsVoidResult.Should().BeTrue(); + called = true; + }); + + _timeProvider.Setup(v => v.UtcNow).Returns(DateTime.UtcNow); + using var controller = CreateController(); + var context = ResilienceContext.Get(); + + // act + await controller.IsolateCircuitAsync(context); + + // assert + controller.CircuitState.Should().Be(CircuitState.Isolated); + called.Should().BeTrue(); + + await Assert.ThrowsAsync(async () => await controller.OnActionPreExecuteAsync(ResilienceContext.Get())); + + // now close it + _circuitBehavior.Setup(v => v.OnCircuitClosed()); + await controller.CloseCircuitAsync(ResilienceContext.Get()); + await controller.OnActionPreExecuteAsync(ResilienceContext.Get()); + context.ResilienceEvents.Should().Contain(new ReportedResilienceEvent("OnCircuitOpened")); + } + + [Fact] + public async Task BreakAsync_Ok() + { + // arrange + bool called = false; + _options.OnClosed.Register((outcome, args) => + { + args.Context.IsSynchronous.Should().BeFalse(); + args.Context.IsVoid.Should().BeTrue(); + args.IsManual.Should().BeTrue(); + outcome.IsVoidResult.Should().BeTrue(); + called = true; + }); + + _timeProvider.Setup(v => v.UtcNow).Returns(DateTime.UtcNow); + using var controller = CreateController(); + await controller.IsolateCircuitAsync(ResilienceContext.Get()); + _circuitBehavior.Setup(v => v.OnCircuitClosed()); + var context = ResilienceContext.Get(); + + // act + await controller.CloseCircuitAsync(context); + + // assert + called.Should().BeTrue(); + + await controller.OnActionPreExecuteAsync(ResilienceContext.Get()); + _circuitBehavior.VerifyAll(); + context.ResilienceEvents.Should().Contain(new ReportedResilienceEvent("OnCircuitClosed")); + } + + [Fact] + public async Task Disposed_EnsureThrows() + { + var controller = CreateController(); + controller.Dispose(); + + Assert.Throws(() => controller.CircuitState); + Assert.Throws(() => controller.LastException); + Assert.Throws(() => controller.LastHandledOutcome); + + await Assert.ThrowsAsync(async () => await controller.CloseCircuitAsync(ResilienceContext.Get())); + await Assert.ThrowsAsync(async () => await controller.IsolateCircuitAsync(ResilienceContext.Get())); + await Assert.ThrowsAsync(async () => await controller.OnActionPreExecuteAsync(ResilienceContext.Get())); + await Assert.ThrowsAsync(async () => await controller.OnActionSuccessAsync(new Outcome(10), ResilienceContext.Get())); + await Assert.ThrowsAsync(async () => await controller.OnActionFailureAsync(new Outcome(10), ResilienceContext.Get())); + } + + [Fact] + public async Task OnActionPreExecute_CircuitOpenedByValue() + { + using var controller = CreateController(); + + await OpenCircuit(controller, new Outcome(99)); + var error = await Assert.ThrowsAsync>(async () => await controller.OnActionPreExecuteAsync(ResilienceContext.Get())); + error.Result.Should().Be(99); + + GetBlockedTill(controller).Should().Be(_utcNow + _options.BreakDuration); + } + + [Fact] + public async Task HalfOpen_EnsureBreakDuration() + { + using var controller = CreateController(); + + await TransitionToState(controller, CircuitState.HalfOpen); + GetBlockedTill(controller).Should().Be(_utcNow + _options.BreakDuration); + } + + [InlineData(true)] + [InlineData(false)] + [Theory] + public async Task HalfOpen_EnsureCorrectStateTransitionAfterExecution(bool success) + { + using var controller = CreateController(); + + await TransitionToState(controller, CircuitState.HalfOpen); + + if (success) + { + _circuitBehavior.Setup(v => v.OnActionSuccess(CircuitState.HalfOpen)); + _circuitBehavior.Setup(v => v.OnCircuitClosed()); + + await controller.OnActionSuccessAsync(new Outcome(0), ResilienceContext.Get()); + controller.CircuitState.Should().Be(CircuitState.Closed); + } + else + { + var shouldBreak = true; + _circuitBehavior.Setup(v => v.OnActionFailure(CircuitState.HalfOpen, out shouldBreak)); + await controller.OnActionFailureAsync(new Outcome(0), ResilienceContext.Get()); + controller.CircuitState.Should().Be(CircuitState.Open); + } + } + + [Fact] + public async Task OnActionPreExecute_CircuitOpenedByException() + { + using var controller = CreateController(); + + await OpenCircuit(controller, new Outcome(new InvalidOperationException())); + var error = await Assert.ThrowsAsync(async () => await controller.OnActionPreExecuteAsync(ResilienceContext.Get())); + error.InnerException.Should().BeOfType(); + } + + [Fact] + public async Task OnActionFailure_EnsureLock() + { + // arrange + using var executing = new ManualResetEvent(false); + using var verified = new ManualResetEvent(false); + + AdvanceTime(_options.BreakDuration); + bool shouldBreak = false; + _circuitBehavior.Setup(v => v.OnActionFailure(CircuitState.Closed, out shouldBreak)).Callback(() => + { + executing.Set(); + verified.WaitOne(); + }); + + using var controller = CreateController(); + + // act + var executeAction = Task.Run(() => controller.OnActionFailureAsync(new Outcome(0), ResilienceContext.Get())); + executing.WaitOne(); + var executeAction2 = Task.Run(() => controller.OnActionFailureAsync(new Outcome(0), ResilienceContext.Get())); + + // assert + executeAction.Wait(50).Should().BeFalse(); + verified.Set(); + await executeAction; + await executeAction2; + } + + [Fact] + public async Task OnActionPreExecute_HalfOpen() + { + // arrange + var called = false; + _options.OnHalfOpened.Register(_ => called = true); + using var controller = CreateController(); + + await OpenCircuit(controller, new Outcome(10)); + AdvanceTime(_options.BreakDuration); + + // act + await controller.OnActionPreExecuteAsync(ResilienceContext.Get()); + var error = await Assert.ThrowsAsync>(async () => await controller.OnActionPreExecuteAsync(ResilienceContext.Get())); + + // assert + controller.CircuitState.Should().Be(CircuitState.HalfOpen); + called.Should().BeTrue(); + } + + [InlineData(CircuitState.HalfOpen, CircuitState.Closed)] + [InlineData(CircuitState.Isolated, CircuitState.Isolated)] + [InlineData(CircuitState.Closed, CircuitState.Closed)] + [Theory] + public async Task OnActionSuccess_EnsureCorrectBehavior(CircuitState state, CircuitState expectedState) + { + // arrange + var called = false; + _options.OnClosed.Register((_, args) => + { + args.IsManual.Should().BeFalse(); + called = true; + }); + using var controller = CreateController(); + + await TransitionToState(controller, state); + + _circuitBehavior.Setup(v => v.OnActionSuccess(state)); + if (expectedState == CircuitState.Closed && state != CircuitState.Closed) + { + _circuitBehavior.Setup(v => v.OnCircuitClosed()); + } + + // act + await controller.OnActionSuccessAsync(new Outcome(10), ResilienceContext.Get()); + + // assert + controller.CircuitState.Should().Be(expectedState); + _circuitBehavior.VerifyAll(); + + if (expectedState == CircuitState.Closed && state != CircuitState.Closed) + { + called.Should().BeTrue(); + } + } + + [InlineData(CircuitState.HalfOpen, CircuitState.Open, true)] + [InlineData(CircuitState.Closed, CircuitState.Open, true)] + [InlineData(CircuitState.Closed, CircuitState.Closed, false)] + [InlineData(CircuitState.Open, CircuitState.Open, false)] + [InlineData(CircuitState.Isolated, CircuitState.Isolated, false)] + [Theory] + public async Task OnActionFailureAsync_EnsureCorrectBehavior(CircuitState state, CircuitState expectedState, bool shouldBreak) + { + // arrange + var called = false; + _options.OnOpened.Register((_, args) => + { + args.IsManual.Should().BeFalse(); + called = true; + }); + using var controller = CreateController(); + + await TransitionToState(controller, state); + _circuitBehavior.Setup(v => v.OnActionFailure(state, out shouldBreak)); + + // act + await controller.OnActionFailureAsync(new Outcome("dummy"), ResilienceContext.Get()); + + // assert + controller.LastHandledOutcome!.Value.Result.Should().Be("dummy"); + controller.CircuitState.Should().Be(expectedState); + _circuitBehavior.VerifyAll(); + + if (expectedState == CircuitState.Open && state != CircuitState.Open) + { + called.Should().BeTrue(); + } + } + + [InlineData(true)] + [InlineData(false)] + [Theory] + public async Task OnActionFailureAsync_EnsureBreakDurationNotOverflow(bool overflow) + { + // arrange + using var controller = CreateController(); + var shouldBreak = true; + await TransitionToState(controller, CircuitState.HalfOpen); + _utcNow = DateTime.MaxValue - _options.BreakDuration; + if (overflow) + { + _utcNow += TimeSpan.FromMilliseconds(10); + } + + _circuitBehavior.Setup(v => v.OnActionFailure(CircuitState.HalfOpen, out shouldBreak)); + + // act + await controller.OnActionFailureAsync(new Outcome("dummy"), ResilienceContext.Get()); + + // assert + var blockedTill = GetBlockedTill(controller); + + if (overflow) + { + blockedTill.Should().Be(DateTimeOffset.MaxValue); + } + else + { + blockedTill.Should().Be(_utcNow + _options.BreakDuration); + } + } + + [Fact] + public async Task OnActionFailureAsync_VoidResult_EnsureBreakingExceptionNotSet() + { + // arrange + using var controller = CreateController(); + bool shouldBreak = true; + await TransitionToState(controller, CircuitState.Open); + _circuitBehavior.Setup(v => v.OnActionFailure(CircuitState.Open, out shouldBreak)); + + // act + await controller.OnActionFailureAsync(new Outcome(VoidResult.Instance), ResilienceContext.Get()); + + // assert + controller.LastException.Should().BeNull(); + await Assert.ThrowsAsync(async () => await controller.OnActionPreExecuteAsync(ResilienceContext.Get())); + } + + [Fact] + public async Task Flow_Closed_HalfOpen_Closed() + { + using var controller = CreateController(); + + await TransitionToState(controller, CircuitState.HalfOpen); + _circuitBehavior.Setup(v => v.OnActionSuccess(CircuitState.HalfOpen)); + _circuitBehavior.Setup(v => v.OnCircuitClosed()); + + await controller.OnActionSuccessAsync(new Outcome(0), ResilienceContext.Get()); + controller.CircuitState.Should().Be(CircuitState.Closed); + } + + [Fact] + public async Task Flow_Closed_HalfOpen_Open_HalfOpen_Closed() + { + var context = ResilienceContext.Get(); + using var controller = CreateController(); + bool shouldBreak = true; + + await TransitionToState(controller, CircuitState.HalfOpen); + + _circuitBehavior.Setup(v => v.OnActionFailure(CircuitState.HalfOpen, out shouldBreak)); + await controller.OnActionFailureAsync(new Outcome(0), context); + controller.CircuitState.Should().Be(CircuitState.Open); + + // execution rejected + AdvanceTime(TimeSpan.FromMilliseconds(1)); + await Assert.ThrowsAsync>(async () => await controller.OnActionPreExecuteAsync(context)); + + // wait and try, transition to half open + AdvanceTime(_options.BreakDuration + _options.BreakDuration); + await controller.OnActionPreExecuteAsync(context); + controller.CircuitState.Should().Be(CircuitState.HalfOpen); + + // close circuit + _circuitBehavior.Setup(v => v.OnActionSuccess(CircuitState.HalfOpen)); + _circuitBehavior.Setup(v => v.OnCircuitClosed()); + await controller.OnActionSuccessAsync(new Outcome(0), ResilienceContext.Get()); + controller.CircuitState.Should().Be(CircuitState.Closed); + } + + private static DateTimeOffset? GetBlockedTill(CircuitStateController controller) => + (DateTimeOffset?)controller.GetType().GetField("_blockedUntil", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(controller)!; + + private async Task TransitionToState(CircuitStateController controller, CircuitState state) + { + switch (state) + { + case CircuitState.Closed: + break; + case CircuitState.Open: + await OpenCircuit(controller); + break; + case CircuitState.HalfOpen: + await OpenCircuit(controller); + AdvanceTime(_options.BreakDuration); + await controller.OnActionPreExecuteAsync(ResilienceContext.Get()); + break; + case CircuitState.Isolated: + await controller.IsolateCircuitAsync(ResilienceContext.Get()); + break; + } + + controller.CircuitState.Should().Be(state); + } + + private async Task OpenCircuit(CircuitStateController controller, Outcome? outcome = null) + { + bool breakCircuit = true; + _circuitBehavior.Setup(v => v.OnActionFailure(CircuitState.Closed, out breakCircuit)); + await controller.OnActionFailureAsync(outcome ?? new Outcome(10), ResilienceContext.Get().Initialize(true)); + } + + private void AdvanceTime(TimeSpan timespan) => _utcNow += timespan; + + private CircuitStateController CreateController() => new( + _options, + _circuitBehavior.Object, + _timeProvider.Object, + TestUtilities.CreateResilienceTelemetry(args => _onTelemetry.Invoke(args))); +} diff --git a/src/Polly.Core.Tests/CircuitBreaker/Controller/ConsecutiveFailuresCircuitBehaviorTests.cs b/src/Polly.Core.Tests/CircuitBreaker/Controller/ConsecutiveFailuresCircuitBehaviorTests.cs new file mode 100644 index 00000000000..10e2e6a9e92 --- /dev/null +++ b/src/Polly.Core.Tests/CircuitBreaker/Controller/ConsecutiveFailuresCircuitBehaviorTests.cs @@ -0,0 +1,49 @@ +using Polly.CircuitBreaker; + +namespace Polly.Core.Tests.CircuitBreaker.Controller; +public class ConsecutiveFailuresCircuitBehaviorTests +{ + [Fact] + public void OnCircuitReset_Ok() + { + var behavior = new ConsecutiveFailuresCircuitBehavior(new CircuitBreakerStrategyOptions { FailureThreshold = 2 }); + + behavior.OnActionFailure(CircuitState.Closed, out var shouldBreak); + behavior.OnCircuitClosed(); + behavior.OnActionFailure(CircuitState.Closed, out shouldBreak); + + shouldBreak.Should().BeFalse(); + } + + [InlineData(1, 1, true)] + [InlineData(2, 1, false)] + [Theory] + public void OnActionFailure_Ok(int threshold, int failures, bool expectedShouldBreak) + { + var behavior = new ConsecutiveFailuresCircuitBehavior(new CircuitBreakerStrategyOptions { FailureThreshold = threshold }); + + for (int i = 0; i < failures - 1; i++) + { + behavior.OnActionFailure(CircuitState.Closed, out _); + } + + behavior.OnActionFailure(CircuitState.Closed, out var shouldBreak); + shouldBreak.Should().Be(expectedShouldBreak); + } + + [InlineData(CircuitState.Closed, false)] + [InlineData(CircuitState.Open, true)] + [InlineData(CircuitState.Isolated, true)] + [InlineData(CircuitState.HalfOpen, true)] + [Theory] + public void OnActionSuccess_Ok(CircuitState state, bool expected) + { + var behavior = new ConsecutiveFailuresCircuitBehavior(new CircuitBreakerStrategyOptions { FailureThreshold = 2 }); + + behavior.OnActionFailure(CircuitState.Closed, out var shouldBreak); + behavior.OnActionSuccess(state); + behavior.OnActionFailure(CircuitState.Closed, out shouldBreak); + + shouldBreak.Should().Be(expected); + } +} diff --git a/src/Polly.Core.Tests/CircuitBreaker/Controller/ScheduledTaskExecutorTests.cs b/src/Polly.Core.Tests/CircuitBreaker/Controller/ScheduledTaskExecutorTests.cs new file mode 100644 index 00000000000..71eccd766c2 --- /dev/null +++ b/src/Polly.Core.Tests/CircuitBreaker/Controller/ScheduledTaskExecutorTests.cs @@ -0,0 +1,149 @@ +using System.Threading.Tasks; +using Polly.CircuitBreaker; + +namespace Polly.Core.Tests.CircuitBreaker.Controller; + +public class ScheduledTaskExecutorTests +{ + [Fact] + public async Task ScheduleTask_Success_EnsureExecuted() + { + using var scheduler = new ScheduledTaskExecutor(); + var executed = false; + scheduler.ScheduleTask( + () => + { + executed = true; + return Task.CompletedTask; + }, + ResilienceContext.Get(), + out var task); + + await task; + + executed.Should().BeTrue(); + } + + [Fact] + public async Task ScheduleTask_OperationCancelledException_EnsureExecuted() + { + using var scheduler = new ScheduledTaskExecutor(); + scheduler.ScheduleTask( + () => throw new OperationCanceledException(), + ResilienceContext.Get(), + out var task); + + await task.Invoking(async t => await task).Should().ThrowAsync(); + } + + [Fact] + public async Task ScheduleTask_Exception_EnsureExecuted() + { + using var scheduler = new ScheduledTaskExecutor(); + scheduler.ScheduleTask( + () => throw new InvalidOperationException(), + ResilienceContext.Get(), + out var task); + + await task.Invoking(async t => await task).Should().ThrowAsync(); + } + + [Fact] + public async Task ScheduleTask_Multiple_EnsureExecutionSerialized() + { + using var executing = new ManualResetEvent(false); + using var verified = new ManualResetEvent(false); + + using var scheduler = new ScheduledTaskExecutor(); + scheduler.ScheduleTask( + () => + { + executing.Set(); + verified.WaitOne(); + return Task.CompletedTask; + }, + ResilienceContext.Get(), + out var task); + + executing.WaitOne(); + + scheduler.ScheduleTask(() => Task.CompletedTask, ResilienceContext.Get(), out var otherTask); + otherTask.Wait(50).Should().BeFalse(); + + verified.Set(); + + await task; + await otherTask; + } + + [Fact] + public async Task Dispose_ScheduledTaskCancelled() + { + using var executing = new ManualResetEvent(false); + using var verified = new ManualResetEvent(false); + + var scheduler = new ScheduledTaskExecutor(); + scheduler.ScheduleTask( + () => + { + executing.Set(); + verified.WaitOne(); + return Task.CompletedTask; + }, + ResilienceContext.Get(), + out var task); + + executing.WaitOne(); + scheduler.ScheduleTask(() => Task.CompletedTask, ResilienceContext.Get(), out var otherTask); + scheduler.Dispose(); + verified.Set(); + await task; + + await otherTask.Invoking(t => otherTask).Should().ThrowAsync(); + + scheduler + .Invoking(s => s.ScheduleTask(() => Task.CompletedTask, ResilienceContext.Get(), out _)) + .Should() + .Throw(); + } + + [Fact] + public void Dispose_WhenScheduledTaskExecuting() + { + using var disposed = new ManualResetEvent(false); + using var ready = new ManualResetEvent(false); + + var scheduler = new ScheduledTaskExecutor(); + scheduler.ScheduleTask( + () => + { + ready.Set(); + disposed.WaitOne(); + return Task.CompletedTask; + }, + ResilienceContext.Get(), + out var task); + + ready.WaitOne(TimeSpan.FromSeconds(2)).Should().BeTrue(); + scheduler.Dispose(); + disposed.Set(); + + scheduler.ProcessingTask.Wait(TimeSpan.FromSeconds(2)).Should().BeTrue(); + } + + [Fact] + public async Task Dispose_EnsureNoBackgroundProcessing() + { + var scheduler = new ScheduledTaskExecutor(); + scheduler.ScheduleTask(() => Task.CompletedTask, ResilienceContext.Get(), out var otherTask); + await otherTask; + scheduler.Dispose(); +#pragma warning disable S3966 // Objects should not be disposed more than once + scheduler.Dispose(); +#pragma warning restore S3966 // Objects should not be disposed more than once + + await scheduler.ProcessingTask; + + scheduler.ProcessingTask.IsCompleted.Should().BeTrue(); + } +} diff --git a/src/Polly.Core.Tests/CircuitBreaker/OnCircuitClosedArgumentsTests.cs b/src/Polly.Core.Tests/CircuitBreaker/OnCircuitClosedArgumentsTests.cs index 316aec95dde..5b323ae97d4 100644 --- a/src/Polly.Core.Tests/CircuitBreaker/OnCircuitClosedArgumentsTests.cs +++ b/src/Polly.Core.Tests/CircuitBreaker/OnCircuitClosedArgumentsTests.cs @@ -9,8 +9,9 @@ public void Ctor_Ok() { var context = ResilienceContext.Get(); - var args = new OnCircuitClosedArguments(context); + var args = new OnCircuitClosedArguments(context, true); args.Context.Should().Be(context); + args.IsManual.Should().BeTrue(); } } diff --git a/src/Polly.Core.Tests/CircuitBreaker/OnCircuitOpenedArgumentsTests.cs b/src/Polly.Core.Tests/CircuitBreaker/OnCircuitOpenedArgumentsTests.cs index 639dd1acea2..02c1bfc2faa 100644 --- a/src/Polly.Core.Tests/CircuitBreaker/OnCircuitOpenedArgumentsTests.cs +++ b/src/Polly.Core.Tests/CircuitBreaker/OnCircuitOpenedArgumentsTests.cs @@ -9,9 +9,10 @@ public void Ctor_Ok() { var context = ResilienceContext.Get(); - var args = new OnCircuitOpenedArguments(context, TimeSpan.FromSeconds(2)); + var args = new OnCircuitOpenedArguments(context, TimeSpan.FromSeconds(2), true); args.Context.Should().Be(context); args.BreakDuration.Should().Be(TimeSpan.FromSeconds(2)); + args.IsManual.Should().BeTrue(); } } diff --git a/src/Polly.Core/CircuitBreaker/BaseCircuitBreakerStrategyOptions.TResult.cs b/src/Polly.Core/CircuitBreaker/BaseCircuitBreakerStrategyOptions.TResult.cs index a12e375b6d6..ccc177903ea 100644 --- a/src/Polly.Core/CircuitBreaker/BaseCircuitBreakerStrategyOptions.TResult.cs +++ b/src/Polly.Core/CircuitBreaker/BaseCircuitBreakerStrategyOptions.TResult.cs @@ -40,18 +40,48 @@ public abstract class BaseCircuitBreakerStrategyOptions : ResilienceStr /// /// Gets or sets the event that is raised when the circuit resets to a state. /// + /// + /// The callbacks registered to this event are invoked with eventual consistency. There is no guarantee that the circuit breaker + /// doesn't change the state before the callbacks finish. If you need to know the up-to-date state of the circuit breaker use + /// the property. + /// + /// Note that these events might be executed asynchronously at a later time when the circuit state is no longer the same as at the point of invocation of the event. + /// However, the invocation order of the , , and events is always + /// maintained to ensure the correct sequence of state transitions. + /// + /// [Required] public OutcomeEvent OnClosed { get; set; } = new(); /// /// Gets or sets the event that is raised when the circuit transitions to an state. /// + /// + /// The callbacks registered to this event are invoked with eventual consistency. There is no guarantee that the circuit breaker + /// doesn't change the state before the callbacks finish. If you need to know the up-to-date state of the circuit breaker use + /// the property. + /// + /// Note that these events might be executed asynchronously at a later time when the circuit state is no longer the same as at the point of invocation of the event. + /// However, the invocation order of the , , and events is always + /// maintained to ensure the correct sequence of state transitions. + /// + /// [Required] public OutcomeEvent OnOpened { get; set; } = new(); /// /// Gets or sets the event that is raised when when the circuit transitions to an state. /// + /// + /// The callbacks registered to this event are invoked with eventual consistency. There is no guarantee that the circuit breaker + /// doesn't change the state before the callbacks finish. If you need to know the up-to-date state of the circuit breaker use + /// the property. + /// + /// Note that these events might be executed asynchronously at a later time when the circuit state is no longer the same as at the point of invocation of the event. + /// However, the invocation order of the , , and events is always + /// maintained to ensure the correct sequence of state transitions. + /// + /// [Required] public NoOutcomeEvent OnHalfOpened { get; set; } = new(); diff --git a/src/Polly.Core/CircuitBreaker/BaseCircuitBreakerStrategyOptions.cs b/src/Polly.Core/CircuitBreaker/BaseCircuitBreakerStrategyOptions.cs index 130ba1f007a..7ebabaed0f2 100644 --- a/src/Polly.Core/CircuitBreaker/BaseCircuitBreakerStrategyOptions.cs +++ b/src/Polly.Core/CircuitBreaker/BaseCircuitBreakerStrategyOptions.cs @@ -39,18 +39,48 @@ public abstract class BaseCircuitBreakerStrategyOptions : ResilienceStrategyOpti /// /// Gets or sets the event that is raised when the circuit resets to a state. /// + /// + /// The callbacks registered to this event are invoked with eventual consistency. There is no guarantee that the circuit breaker + /// doesn't change the state before the callbacks finish. If you need to know the up-to-date state of the circuit breaker use + /// the property. + /// + /// Note that these events might be executed asynchronously at a later time when the circuit state is no longer the same as at the point of invocation of the event. + /// However, the invocation order of the , , and events is always + /// maintained to ensure the correct sequence of state transitions. + /// + /// [Required] public OutcomeEvent OnClosed { get; set; } = new(); /// /// Gets or sets the event that is raised when the circuit transitions to an state. /// + /// + /// The callbacks registered to this event are invoked with eventual consistency. There is no guarantee that the circuit breaker + /// doesn't change the state before the callbacks finish. If you need to know the up-to-date state of the circuit breaker use + /// the property. + /// + /// Note that these events might be executed asynchronously at a later time when the circuit state is no longer the same as at the point of invocation of the event. + /// However, the invocation order of the , , and events is always + /// maintained to ensure the correct sequence of state transitions. + /// + /// [Required] public OutcomeEvent OnOpened { get; set; } = new(); /// /// Gets or sets the event that is raised when when the circuit transitions to an state. /// + /// + /// The callbacks registered to this event are invoked with eventual consistency. There is no guarantee that the circuit breaker + /// doesn't change the state before the callbacks finish. If you need to know the up-to-date state of the circuit breaker use + /// the property. + /// + /// Note that these events might be executed asynchronously at a later time when the circuit state is no longer the same as at the point of invocation of the event. + /// However, the invocation order of the , , and events is always + /// maintained to ensure the correct sequence of state transitions. + /// + /// [Required] public NoOutcomeEvent OnHalfOpened { get; set; } = new(); diff --git a/src/Polly.Core/CircuitBreaker/BrokenCircuitException.cs b/src/Polly.Core/CircuitBreaker/BrokenCircuitException.cs index 357a0d97a9e..9923a6b2f3b 100644 --- a/src/Polly.Core/CircuitBreaker/BrokenCircuitException.cs +++ b/src/Polly.Core/CircuitBreaker/BrokenCircuitException.cs @@ -12,10 +12,13 @@ namespace Polly.CircuitBreaker; #endif public class BrokenCircuitException : ExecutionRejectedException { + internal const string DefaultMessage = "The circuit is now open and is not allowing calls."; + /// /// Initializes a new instance of the class. /// public BrokenCircuitException() + : base(DefaultMessage) { } diff --git a/src/Polly.Core/CircuitBreaker/CircuitBreakerConstants.cs b/src/Polly.Core/CircuitBreaker/CircuitBreakerConstants.cs index 3a5835c2be8..fd162b10447 100644 --- a/src/Polly.Core/CircuitBreaker/CircuitBreakerConstants.cs +++ b/src/Polly.Core/CircuitBreaker/CircuitBreakerConstants.cs @@ -4,11 +4,11 @@ internal static class CircuitBreakerConstants { public const string StrategyType = "CircuitBreaker"; - public const string OnResetEvent = "OnCircuitReset"; + public const string OnCircuitClosed = "OnCircuitClosed"; - public const string OnHalfOpenEvent = "OnCircuitHalfOpen"; + public const string OnHalfOpenEvent = "OnCircuitHalfOpened"; - public const string OnBreakEvent = "OnCircuitBreak"; + public const string OnCircuitOpened = "OnCircuitOpened"; public const double DefaultAdvancedFailureThreshold = 0.1; diff --git a/src/Polly.Core/CircuitBreaker/CircuitBreakerManualControl.cs b/src/Polly.Core/CircuitBreaker/CircuitBreakerManualControl.cs index 498a1797e9c..68cd0c616ca 100644 --- a/src/Polly.Core/CircuitBreaker/CircuitBreakerManualControl.cs +++ b/src/Polly.Core/CircuitBreaker/CircuitBreakerManualControl.cs @@ -5,18 +5,20 @@ namespace Polly.CircuitBreaker; /// /// Allows manual control of the circuit-breaker. /// -public sealed class CircuitBreakerManualControl +public sealed class CircuitBreakerManualControl : IDisposable { + private Action? _onDispose; private Func? _onIsolate; private Func? _onReset; - internal void Initialize(Func onIsolate, Func onReset) + internal void Initialize(Func onIsolate, Func onReset, Action onDispose) { if (_onIsolate != null) { throw new InvalidOperationException($"This instance of '{nameof(CircuitBreakerManualControl)}' is already initialized and cannot be used in a different circuit-breaker strategy."); } + _onDispose = onDispose; _onIsolate = onIsolate; _onReset = onReset; } @@ -31,7 +33,7 @@ internal void Initialize(Func onIsolate, Func _onIsolate != null; /// - /// Isolates (opens) the circuit manually, and holds it in this state until a call to is made. + /// Isolates (opens) the circuit manually, and holds it in this state until a call to is made. /// /// The resilience context. /// The instance of that represents the asynchronous execution. @@ -50,7 +52,7 @@ public Task IsolateAsync(ResilienceContext context) } /// - /// Isolates (opens) the circuit manually, and holds it in this state until a call to is made. + /// Isolates (opens) the circuit manually, and holds it in this state until a call to is made. /// /// The cancellation token. /// The instance of that represents the asynchronous execution. @@ -76,7 +78,7 @@ public async Task IsolateAsync(CancellationToken cancellationToken) /// The resilience context. /// The instance of that represents the asynchronous execution. /// Thrown if manual control is not initialized. - public Task ResetAsync(ResilienceContext context) + public Task CloseAsync(ResilienceContext context) { Guard.NotNull(context); @@ -95,18 +97,21 @@ public Task ResetAsync(ResilienceContext context) /// The cancellation token. /// The instance of that represents the asynchronous execution. /// Thrown if manual control is not initialized. - public async Task ResetAsync(CancellationToken cancellationToken) + public async Task CloseAsync(CancellationToken cancellationToken) { var context = ResilienceContext.Get(); context.CancellationToken = cancellationToken; try { - await ResetAsync(context).ConfigureAwait(false); + await CloseAsync(context).ConfigureAwait(false); } finally { ResilienceContext.Return(context); } } + + /// + public void Dispose() => _onDispose?.Invoke(); } diff --git a/src/Polly.Core/CircuitBreaker/CircuitBreakerResilienceStrategy.cs b/src/Polly.Core/CircuitBreaker/CircuitBreakerResilienceStrategy.cs index 6f702d0464b..667b8a2a600 100644 --- a/src/Polly.Core/CircuitBreaker/CircuitBreakerResilienceStrategy.cs +++ b/src/Polly.Core/CircuitBreaker/CircuitBreakerResilienceStrategy.cs @@ -4,22 +4,69 @@ namespace Polly.CircuitBreaker; internal sealed class CircuitBreakerResilienceStrategy : ResilienceStrategy { -#pragma warning disable IDE0052 // Remove unread private members - private readonly TimeProvider _timeProvider; - private readonly ResilienceStrategyTelemetry _telemetry; - private readonly BaseCircuitBreakerStrategyOptions _options; -#pragma warning restore IDE0052 // Remove unread private members + private readonly CircuitStateController _controller; + private readonly OutcomePredicate.Handler? _handler; - public CircuitBreakerResilienceStrategy(TimeProvider timeProvider, ResilienceStrategyTelemetry telemetry, BaseCircuitBreakerStrategyOptions options) + public CircuitBreakerResilienceStrategy(BaseCircuitBreakerStrategyOptions options, CircuitStateController controller) { - _timeProvider = timeProvider; - _telemetry = telemetry; - _options = options; + _controller = controller; + _handler = options.ShouldHandle.CreateHandler(); + + options.StateProvider?.Initialize(() => _controller.CircuitState, () => _controller.LastHandledOutcome); + options.ManualControl?.Initialize( + async c => await _controller.IsolateCircuitAsync(c).ConfigureAwait(c.ContinueOnCapturedContext), + async c => await _controller.CloseCircuitAsync(c).ConfigureAwait(c.ContinueOnCapturedContext), + _controller.Dispose); } - protected internal override ValueTask ExecuteCoreAsync(Func> callback, ResilienceContext context, TState state) + protected internal override async ValueTask ExecuteCoreAsync(Func> callback, ResilienceContext context, TState state) { - return callback(context, state); + if (_handler == null) + { + return await callback(context, state).ConfigureAwait(context.ContinueOnCapturedContext); + } + + await _controller.OnActionPreExecuteAsync(context).ConfigureAwait(context.ContinueOnCapturedContext); + + try + { + var result = await callback(context, state).ConfigureAwait(context.ContinueOnCapturedContext); + + await HandleResultAsync(context, result).ConfigureAwait(context.ContinueOnCapturedContext); + + return result; + } + catch (Exception e) + { + await HandleExceptionAsync(context, e).ConfigureAwait(context.ContinueOnCapturedContext); + + throw; + } + } + + private async Task HandleResultAsync(ResilienceContext context, TResult result) + { + var outcome = new Outcome(result); + var args = new CircuitBreakerPredicateArguments(context); + if (await _handler!.ShouldHandleAsync(outcome, args).ConfigureAwait(context.ContinueOnCapturedContext)) + { + await _controller.OnActionFailureAsync(outcome, context).ConfigureAwait(context.ContinueOnCapturedContext); + } + else + { + await _controller.OnActionSuccessAsync(outcome, context).ConfigureAwait(context.ContinueOnCapturedContext); + } + } + + private async Task HandleExceptionAsync(ResilienceContext context, Exception e) + { + var args = new CircuitBreakerPredicateArguments(context); + var outcome = new Outcome(e); + + if (await _handler!.ShouldHandleAsync(outcome, args).ConfigureAwait(context.ContinueOnCapturedContext)) + { + await _controller.OnActionFailureAsync(outcome, context).ConfigureAwait(context.ContinueOnCapturedContext); + } } } diff --git a/src/Polly.Core/CircuitBreaker/CircuitBreakerResilienceStrategyBuilderExtensions.cs b/src/Polly.Core/CircuitBreaker/CircuitBreakerResilienceStrategyBuilderExtensions.cs index 67ab92afa5f..2a4cc556c5d 100644 --- a/src/Polly.Core/CircuitBreaker/CircuitBreakerResilienceStrategyBuilderExtensions.cs +++ b/src/Polly.Core/CircuitBreaker/CircuitBreakerResilienceStrategyBuilderExtensions.cs @@ -17,6 +17,9 @@ public static class CircuitBreakerResilienceStrategyBuilderExtensions /// A builder with the circuit breaker strategy added. /// /// See for more details about the advanced circuit breaker strategy. + /// + /// If you are discarding the strategy created by this call make sure to use and dispose the manual control instance when the strategy is no longer used. + /// /// public static ResilienceStrategyBuilder AddAdvancedCircuitBreaker(this ResilienceStrategyBuilder builder, AdvancedCircuitBreakerStrategyOptions options) { @@ -36,6 +39,9 @@ public static ResilienceStrategyBuilder AddAdvancedCircuitBreaker(this /// A builder with the circuit breaker strategy added. /// /// See for more details about the advanced circuit breaker strategy. + /// + /// If you are discarding the strategy created by this call make sure to use and dispose the manual control instance when the strategy is no longer used. + /// /// public static ResilienceStrategyBuilder AddAdvancedCircuitBreaker(this ResilienceStrategyBuilder builder, AdvancedCircuitBreakerStrategyOptions options) { @@ -56,6 +62,9 @@ public static ResilienceStrategyBuilder AddAdvancedCircuitBreaker(this Resilienc /// A builder with the circuit breaker strategy added. /// /// See for more details about the advanced circuit breaker strategy. + /// + /// If you are discarding the strategy created by this call make sure to use and dispose the manual control instance when the strategy is no longer used. + /// /// public static ResilienceStrategyBuilder AddCircuitBreaker(this ResilienceStrategyBuilder builder, CircuitBreakerStrategyOptions options) { @@ -75,6 +84,9 @@ public static ResilienceStrategyBuilder AddCircuitBreaker(this Resilien /// A builder with the circuit breaker strategy added. /// /// See for more details about the advanced circuit breaker strategy. + /// + /// If you are discarding the strategy created by this call make sure to use and dispose the manual control instance when the strategy is no longer used. + /// /// public static ResilienceStrategyBuilder AddCircuitBreaker(this ResilienceStrategyBuilder builder, CircuitBreakerStrategyOptions options) { @@ -86,9 +98,21 @@ public static ResilienceStrategyBuilder AddCircuitBreaker(this ResilienceStrateg return builder.AddCircuitBreakerCore(options); } - private static ResilienceStrategyBuilder AddCircuitBreakerCore(this ResilienceStrategyBuilder builder, BaseCircuitBreakerStrategyOptions options) + internal static ResilienceStrategyBuilder AddCircuitBreakerCore(this ResilienceStrategyBuilder builder, BaseCircuitBreakerStrategyOptions options) { - return builder.AddStrategy(context => new CircuitBreakerResilienceStrategy(context.TimeProvider, context.Telemetry, options)); + return builder.AddStrategy(context => + { + CircuitBehavior behavior = options switch + { + AdvancedCircuitBreakerStrategyOptions => new AdvancedCircuitBehavior(), + CircuitBreakerStrategyOptions o => new ConsecutiveFailuresCircuitBehavior(o), + _ => throw new NotSupportedException() + }; + + var controller = new CircuitStateController(options, behavior, context.TimeProvider, context.Telemetry); + + return new CircuitBreakerResilienceStrategy(options, controller); + }); } } diff --git a/src/Polly.Core/CircuitBreaker/CircuitBreakerStateProvider.cs b/src/Polly.Core/CircuitBreaker/CircuitBreakerStateProvider.cs index 13c87fcd2d7..03e59e235be 100644 --- a/src/Polly.Core/CircuitBreaker/CircuitBreakerStateProvider.cs +++ b/src/Polly.Core/CircuitBreaker/CircuitBreakerStateProvider.cs @@ -1,3 +1,5 @@ +using Polly.Strategy; + namespace Polly.CircuitBreaker; /// @@ -6,9 +8,9 @@ namespace Polly.CircuitBreaker; public sealed class CircuitBreakerStateProvider { private Func? _circuitStateProvider; - private Func? _lastExceptionProvider; + private Func? _lastHandledOutcomeProvider; - internal void Initialize(Func circuitStateProvider, Func lastExceptionProvider) + internal void Initialize(Func circuitStateProvider, Func lastHandledOutcomeProvider) { if (_circuitStateProvider != null) { @@ -16,7 +18,7 @@ internal void Initialize(Func circuitStateProvider, Func @@ -34,8 +36,9 @@ internal void Initialize(Func circuitStateProvider, Func _circuitStateProvider?.Invoke() ?? CircuitState.Closed; /// - /// Gets the last exception handled by the circuit-breaker. - /// This will be null if no exceptions have been handled by the circuit-breaker since the circuit last closed. + /// Gets the last outcome handled by the circuit-breaker. + /// + /// This will be null if no exceptions or results have been handled by the circuit-breaker since the circuit last closed. /// - public Exception? LastException => _lastExceptionProvider?.Invoke(); + public Outcome? LastHandledOutcome => _lastHandledOutcomeProvider?.Invoke(); } diff --git a/src/Polly.Core/CircuitBreaker/Controller/AdvancedCircuitBehavior.cs b/src/Polly.Core/CircuitBreaker/Controller/AdvancedCircuitBehavior.cs new file mode 100644 index 00000000000..c5c5e92d266 --- /dev/null +++ b/src/Polly.Core/CircuitBreaker/Controller/AdvancedCircuitBehavior.cs @@ -0,0 +1,18 @@ +namespace Polly.CircuitBreaker; + +internal sealed class AdvancedCircuitBehavior : CircuitBehavior +{ + public override void OnActionSuccess(CircuitState currentState) + { + } + + public override void OnActionFailure(CircuitState currentState, out bool shouldBreak) + { + shouldBreak = false; + } + + public override void OnCircuitClosed() + { + } +} + diff --git a/src/Polly.Core/CircuitBreaker/Controller/CircuitBehavior.cs b/src/Polly.Core/CircuitBreaker/Controller/CircuitBehavior.cs new file mode 100644 index 00000000000..d1e1f2ee710 --- /dev/null +++ b/src/Polly.Core/CircuitBreaker/Controller/CircuitBehavior.cs @@ -0,0 +1,13 @@ +namespace Polly.CircuitBreaker; + +/// +/// Defines the behavior of circuit breaker. All methods on this class are performed under a lock. +/// +internal abstract class CircuitBehavior +{ + public abstract void OnActionSuccess(CircuitState currentState); + + public abstract void OnActionFailure(CircuitState currentState, out bool shouldBreak); + + public abstract void OnCircuitClosed(); +} diff --git a/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs b/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs new file mode 100644 index 00000000000..70fccc1b076 --- /dev/null +++ b/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs @@ -0,0 +1,326 @@ +using Polly.Strategy; + +namespace Polly.CircuitBreaker; + +/// +/// Thread-safe controller that holds and manages the circuit breaker state transitions. +/// +internal sealed class CircuitStateController : IDisposable +{ + private readonly object _lock = new(); + private readonly ScheduledTaskExecutor _executor = new(); + private readonly OutcomeEvent.Handler? _onOpened; + private readonly OutcomeEvent.Handler? _onClosed; + private readonly Func? _onHalfOpen; + private readonly TimeProvider _timeProvider; + private readonly ResilienceStrategyTelemetry _telemetry; + private readonly CircuitBehavior _behavior; + private readonly TimeSpan _breakDuration; + private DateTimeOffset _blockedUntil; + private CircuitState _circuitState = CircuitState.Closed; + private Outcome? _lastOutcome; + private BrokenCircuitException? _breakingException; + private bool _disposed; + + public CircuitStateController(BaseCircuitBreakerStrategyOptions options, CircuitBehavior behavior, TimeProvider timeProvider, ResilienceStrategyTelemetry telemetry) + { + _breakDuration = options.BreakDuration; + _onOpened = options.OnOpened.CreateHandler(); + _onClosed = options.OnClosed.CreateHandler(); + _onHalfOpen = options.OnHalfOpened.CreateHandler(); + _behavior = behavior; + _timeProvider = timeProvider; + _telemetry = telemetry; + } + + public CircuitState CircuitState + { + get + { + EnsureNotDisposed(); + + lock (_lock) + { + return _circuitState; + } + } + } + + public Exception? LastException + { + get + { + EnsureNotDisposed(); + + lock (_lock) + { + return _lastOutcome?.Exception; + } + } + } + + public Outcome? LastHandledOutcome + { + get + { + EnsureNotDisposed(); + + lock (_lock) + { + return _lastOutcome; + } + } + } + + public ValueTask IsolateCircuitAsync(ResilienceContext context) + { + EnsureNotDisposed(); + + context.Initialize(isSynchronous: false); + + Task? task; + + lock (_lock) + { + SetLastHandledOutcome_NeedsLock(new Outcome(new IsolatedCircuitException())); + OpenCircuitFor_NeedsLock(new Outcome(VoidResult.Instance), TimeSpan.MaxValue, manual: true, context, out task); + _circuitState = CircuitState.Isolated; + } + + return ExecuteScheduledTaskAsync(task, context); + } + + public ValueTask CloseCircuitAsync(ResilienceContext context) + { + EnsureNotDisposed(); + + context.Initialize(isSynchronous: false); + + Task? task; + + lock (_lock) + { + CloseCircuit_NeedsLock(new Outcome(VoidResult.Instance), manual: true, context, out task); + } + + return ExecuteScheduledTaskAsync(task, context); + } + + public async ValueTask OnActionPreExecuteAsync(ResilienceContext context) + { + EnsureNotDisposed(); + + Exception? exception = null; + bool isHalfOpen = false; + + Task? task = null; + + lock (_lock) + { + // check if circuit can be half-opened + if (_circuitState == CircuitState.Open && PermitHalfOpenCircuitTest_NeedsLock()) + { + _circuitState = CircuitState.HalfOpen; + _telemetry.Report(CircuitBreakerConstants.OnHalfOpenEvent, new OnCircuitHalfOpenedArguments(context)); + isHalfOpen = true; + } + + exception = _circuitState switch + { + CircuitState.Open => GetBreakingException_NeedsLock(), + CircuitState.HalfOpen when isHalfOpen is false => GetBreakingException_NeedsLock(), + CircuitState.Isolated => new IsolatedCircuitException(), + _ => null + }; + + if (isHalfOpen && _onHalfOpen is not null) + { + _executor.ScheduleTask(() => _onHalfOpen(new OnCircuitHalfOpenedArguments(context)).AsTask(), context, out task); + } + } + + await ExecuteScheduledTaskAsync(task, context).ConfigureAwait(context.ContinueOnCapturedContext); + + if (exception is not null) + { + throw exception; + } + } + + public ValueTask OnActionSuccessAsync(Outcome outcome, ResilienceContext context) + { + EnsureNotDisposed(); + + Task? task = null; + + lock (_lock) + { + _behavior.OnActionSuccess(_circuitState); + + // Circuit state handling: + // + // HalfOpen - close the circuit + // Closed - do nothing + // Open, Isolated - A successful call result may arrive when the circuit is open, if it was placed before the circuit broke. + // We take no special action; only time passing governs transitioning from Open to HalfOpen state. + if (_circuitState == CircuitState.HalfOpen) + { + CloseCircuit_NeedsLock(outcome, manual: false, context, out task); + } + + } + + return ExecuteScheduledTaskAsync(task, context); + } + + public ValueTask OnActionFailureAsync(Outcome outcome, ResilienceContext context) + { + EnsureNotDisposed(); + + Task? task = null; + + lock (_lock) + { + SetLastHandledOutcome_NeedsLock(outcome); + + _behavior.OnActionFailure(_circuitState, out var shouldBreak); + + // Circuit state handling + // HalfOpen - open the circuit again + // Closed - break the circuit if the behavior indicates it + // Open, Isolated - a failure call result may arrive when the circuit is open, + // if it was placed before the circuit broke. We take no action beyond tracking + // the metric; we do not want to duplicate-signal onBreak; we do not want to extend time for which the circuit is broken. + // We do not want to mask the fact that the call executed (as replacing its result with a Broken/IsolatedCircuitException would do). + + if (_circuitState == CircuitState.HalfOpen) + { + OpenCircuit_NeedsLock(outcome, manual: false, context, out task); + } + else if (_circuitState == CircuitState.Closed && shouldBreak) + { + OpenCircuit_NeedsLock(outcome, manual: false, context, out task); + } + } + + return ExecuteScheduledTaskAsync(task, context); + } + + public void Dispose() + { + _executor.Dispose(); + _disposed = true; + } + + private static async ValueTask ExecuteScheduledTaskAsync(Task? task, ResilienceContext context) + { + if (task is not null) + { + if (context.IsSynchronous) + { +#pragma warning disable CA1849 // Call async methods when in an async method + // because this is synchronous execution we need to block + task.GetAwaiter().GetResult(); +#pragma warning restore CA1849 // Call async methods when in an async method + } + else + { + await task.ConfigureAwait(context.ContinueOnCapturedContext); + } + } + } + + private static bool IsDateTimeOverflow(DateTimeOffset utcNow, TimeSpan breakDuration) + { + TimeSpan maxDifference = DateTime.MaxValue - utcNow; + + // stryker disable once equality : no means to test this + return breakDuration > maxDifference; + } + + private void EnsureNotDisposed() + { + if (_disposed) + { + throw new ObjectDisposedException(nameof(CircuitStateController)); + } + } + + private void CloseCircuit_NeedsLock(Outcome outcome, bool manual, ResilienceContext context, out Task? scheduledTask) + { + scheduledTask = null; + + _blockedUntil = DateTimeOffset.MinValue; + _lastOutcome = null; + _breakingException = null; + + CircuitState priorState = _circuitState; + _circuitState = CircuitState.Closed; + _behavior.OnCircuitClosed(); + + if (priorState != CircuitState.Closed) + { + var args = new OnCircuitClosedArguments(context, manual); + _telemetry.Report(CircuitBreakerConstants.OnCircuitClosed, outcome, args); + + if (_onClosed is not null) + { + _executor.ScheduleTask(() => _onClosed.HandleAsync(outcome, args).AsTask(), context, out scheduledTask); + } + } + } + + private bool PermitHalfOpenCircuitTest_NeedsLock() + { + var now = _timeProvider.UtcNow; + if (now >= _blockedUntil) + { + _blockedUntil = now + _breakDuration; + return true; + } + + return false; + } + + private void SetLastHandledOutcome_NeedsLock(Outcome outcome) + { + _lastOutcome = outcome.AsOutcome(); + _breakingException = null; + + if (outcome.Exception is Exception exception) + { + _breakingException = new BrokenCircuitException(BrokenCircuitException.DefaultMessage, exception); + } + else if (outcome.TryGetResult(out var result)) + { + _breakingException = new BrokenCircuitException(BrokenCircuitException.DefaultMessage, result!); + } + } + + private BrokenCircuitException GetBreakingException_NeedsLock() => _breakingException ?? new BrokenCircuitException(); + + private void OpenCircuit_NeedsLock(Outcome outcome, bool manual, ResilienceContext context, out Task? scheduledTask) + { + OpenCircuitFor_NeedsLock(outcome, _breakDuration, manual, context, out scheduledTask); + } + + private void OpenCircuitFor_NeedsLock(Outcome outcome, TimeSpan breakDuration, bool manual, ResilienceContext context, out Task? scheduledTask) + { + scheduledTask = null; + var utcNow = _timeProvider.UtcNow; + + _blockedUntil = IsDateTimeOverflow(utcNow, breakDuration) ? DateTimeOffset.MaxValue : utcNow + breakDuration; + + var transitionedState = _circuitState; + _circuitState = CircuitState.Open; + + var args = new OnCircuitOpenedArguments(context, breakDuration, manual); + _telemetry.Report(CircuitBreakerConstants.OnCircuitOpened, outcome, args); + + if (_onOpened is not null) + { + _executor.ScheduleTask(() => _onOpened.HandleAsync(outcome, args).AsTask(), context, out scheduledTask); + } + } +} + diff --git a/src/Polly.Core/CircuitBreaker/Controller/ConsecutiveFailuresCircuitBehavior.cs b/src/Polly.Core/CircuitBreaker/Controller/ConsecutiveFailuresCircuitBehavior.cs new file mode 100644 index 00000000000..425fc7164fe --- /dev/null +++ b/src/Polly.Core/CircuitBreaker/Controller/ConsecutiveFailuresCircuitBehavior.cs @@ -0,0 +1,37 @@ +namespace Polly.CircuitBreaker; + +internal sealed class ConsecutiveFailuresCircuitBehavior : CircuitBehavior +{ + private readonly int _failureThreshold; + private int _consecutiveFailures; + + public ConsecutiveFailuresCircuitBehavior(CircuitBreakerStrategyOptions options) => _failureThreshold = options.FailureThreshold; + + public override void OnActionSuccess(CircuitState currentState) + { + if (currentState == CircuitState.Closed) + { + _consecutiveFailures = 0; + } + } + + public override void OnActionFailure(CircuitState currentState, out bool shouldBreak) + { + shouldBreak = false; + + if (currentState == CircuitState.Closed) + { + _consecutiveFailures += 1; + if (_consecutiveFailures >= _failureThreshold) + { + shouldBreak = true; + } + } + } + + public override void OnCircuitClosed() + { + _consecutiveFailures = 0; + } +} + diff --git a/src/Polly.Core/CircuitBreaker/Controller/ScheduledTaskExecutor.cs b/src/Polly.Core/CircuitBreaker/Controller/ScheduledTaskExecutor.cs new file mode 100644 index 00000000000..df2009766de --- /dev/null +++ b/src/Polly.Core/CircuitBreaker/Controller/ScheduledTaskExecutor.cs @@ -0,0 +1,87 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace Polly.CircuitBreaker; + +#pragma warning disable CA1031 // Do not catch general exception types + +/// +/// The scheduled task executor makes sure that tasks are executed in the order they were scheduled and not concurrently. +/// +internal sealed class ScheduledTaskExecutor : IDisposable +{ + private readonly ConcurrentQueue _tasks = new(); + private readonly SemaphoreSlim _semaphore = new(0); + private bool _disposed; + + public ScheduledTaskExecutor() => ProcessingTask = Task.Run(StartProcessingAsync); + + public Task ProcessingTask { get; } + + public void ScheduleTask(Func taskFactory, ResilienceContext context, out Task task) + { + if (_disposed) + { + throw new ObjectDisposedException(nameof(ScheduledTaskExecutor)); + } + + var source = new TaskCompletionSource(); + task = source.Task; + + _tasks.Enqueue(new Entry(taskFactory, context.ContinueOnCapturedContext, source)); + _semaphore.Release(); + } + + public void Dispose() + { + if (_disposed) + { + return; + } + + _disposed = true; + _semaphore.Release(); + + while (_tasks.TryDequeue(out var e)) + { + e.TaskCompletion.TrySetCanceled(); + } + + _semaphore.Dispose(); + } + + private async Task StartProcessingAsync() + { + while (true) + { + await _semaphore.WaitAsync().ConfigureAwait(false); + if (_disposed) + { + return; + } + + _ = _tasks.TryDequeue(out var entry); + + try + { + await entry!.TaskFactory().ConfigureAwait(entry.ContinueOnCapturedContext); + entry.TaskCompletion.SetResult(null!); + } + catch (OperationCanceledException) + { + entry!.TaskCompletion.SetCanceled(); + } + catch (Exception e) + { + entry!.TaskCompletion.SetException(e); + } + + if (_disposed) + { + return; + } + } + } + + private record Entry(Func TaskFactory, bool ContinueOnCapturedContext, TaskCompletionSource TaskCompletion); +} diff --git a/src/Polly.Core/CircuitBreaker/OnCircuitClosedArguments.cs b/src/Polly.Core/CircuitBreaker/OnCircuitClosedArguments.cs index 3e8a7f11dee..3f22be6d00a 100644 --- a/src/Polly.Core/CircuitBreaker/OnCircuitClosedArguments.cs +++ b/src/Polly.Core/CircuitBreaker/OnCircuitClosedArguments.cs @@ -1,4 +1,4 @@ -using Polly.Strategy; +using Polly.Strategy; namespace Polly.CircuitBreaker; @@ -9,7 +9,16 @@ namespace Polly.CircuitBreaker; /// public readonly struct OnCircuitClosedArguments : IResilienceArguments { - internal OnCircuitClosedArguments(ResilienceContext context) => Context = context; + internal OnCircuitClosedArguments(ResilienceContext context, bool isManual) + { + Context = context; + IsManual = isManual; + } + + /// + /// Gets a value indicating whether the circuit was closed manually by using . + /// + public bool IsManual { get; } /// public ResilienceContext Context { get; } diff --git a/src/Polly.Core/CircuitBreaker/OnCircuitOpenedArguments.cs b/src/Polly.Core/CircuitBreaker/OnCircuitOpenedArguments.cs index dbfaf80d5d0..25789f9c972 100644 --- a/src/Polly.Core/CircuitBreaker/OnCircuitOpenedArguments.cs +++ b/src/Polly.Core/CircuitBreaker/OnCircuitOpenedArguments.cs @@ -9,9 +9,10 @@ namespace Polly.CircuitBreaker; /// public readonly struct OnCircuitOpenedArguments : IResilienceArguments { - internal OnCircuitOpenedArguments(ResilienceContext context, TimeSpan breakDuration) + internal OnCircuitOpenedArguments(ResilienceContext context, TimeSpan breakDuration, bool isManual) { BreakDuration = breakDuration; + IsManual = isManual; Context = context; } @@ -20,6 +21,11 @@ internal OnCircuitOpenedArguments(ResilienceContext context, TimeSpan breakDurat /// public TimeSpan BreakDuration { get; } + /// + /// Gets a value indicating whether the circuit was opened manually by using . + /// + public bool IsManual { get; } + /// public ResilienceContext Context { get; } } diff --git a/src/Polly.TestUtils/TestUtilities.cs b/src/Polly.TestUtils/TestUtilities.cs index f6976353500..f28c4ee7044 100644 --- a/src/Polly.TestUtils/TestUtilities.cs +++ b/src/Polly.TestUtils/TestUtilities.cs @@ -41,6 +41,9 @@ public static async Task AssertWithTimeoutAsync(Func assertion, TimeSpan t public static ResilienceStrategyTelemetry CreateResilienceTelemetry(DiagnosticSource source) => new(new ResilienceTelemetrySource("dummy-builder", new ResilienceProperties(), "strategy-name", "strategy-type"), source); + public static ResilienceStrategyTelemetry CreateResilienceTelemetry(Action callback) + => new(new ResilienceTelemetrySource("dummy-builder", new ResilienceProperties(), "strategy-name", "strategy-type"), new CallbackDiagnosticSource(callback)); + public static ILoggerFactory CreateLoggerFactory(out FakeLogger logger) { logger = new FakeLogger(); @@ -93,4 +96,15 @@ public static ResilienceContext WithResultType(this ResilienceContext context context.Initialize(true); return context; } + + private sealed class CallbackDiagnosticSource : DiagnosticSource + { + private readonly Action _callback; + + public CallbackDiagnosticSource(Action callback) => _callback = callback; + + public override bool IsEnabled(string name) => true; + + public override void Write(string name, object? value) => _callback((value as TelemetryEventArguments)!.Arguments); + } }