Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Bump Halibut to fix a polling tentacle cooperative cancellation issue causing request to time out to quickly #695

Closed
wants to merge 1 commit into from
Closed

Bump Halibut to fix a polling tentacle cooperative cancellation issue causing request to time out to quickly #695

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion source/Octopus.Tentacle.Contracts/Octopus.Tentacle.Contracts.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
</Otherwise>
</Choose>
<ItemGroup>
<PackageReference Include="Halibut" Version="7.0.256" />
<PackageReference Include="Halibut" Version="7.0.270-pull-558" />
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TODO - Update when OctopusDeploy/Halibut#558 merges

<PackageReference Include="Octopus.Diagnostics" Version="2.1.0" />
</ItemGroup>
<ItemGroup>
Expand Down
133 changes: 133 additions & 0 deletions source/Octopus.Tentacle.Tests.Integration/PollingRequestAndResponseTimeouts.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
using System;
using System.Threading.Tasks;
using FluentAssertions;
using Halibut;
using NUnit.Framework;
using Octopus.Tentacle.CommonTestUtils.Builders;
using Octopus.Tentacle.Contracts;
using Octopus.Tentacle.Contracts.ClientServices;
using Octopus.Tentacle.Tests.Integration.Support;
using Octopus.Tentacle.Tests.Integration.Util;
using Octopus.Tentacle.Tests.Integration.Util.Builders;
using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators;
using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators.Proxies;

namespace Octopus.Tentacle.Tests.Integration
{
[IntegrationTestTimeout]
public class PollingRequestAndResponseTimeouts : IntegrationTest
{
[Test]
[TentacleConfigurations(testListening: false)]
public async Task WhenThePollingRequestQueueTimeoutIsReached_ForARequestThatHasNotBeenDequeuedByTentacle_TheRequestShouldTimeout(TentacleConfigurationTestCase tentacleConfigurationTestCase)
{
await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder()
.WithRetriesDisabled()
.WithPortForwarder()
.WithServiceEndpointModifier(serviceEndpoint =>
{
serviceEndpoint.PollingRequestQueueTimeout = TimeSpan.FromSeconds(5);
})
.WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder()
.RecordMethodUsages<IAsyncClientCapabilitiesServiceV2>(out var capabilitiesMethodUsages)
.RecordMethodUsages(tentacleConfigurationTestCase, out var scriptServiceMethodUsages)
.Build())
.Build(CancellationToken);

// Stop the request from being dequeued by stopping the Tentacle from connecting
clientTentacle.PortForwarder!.EnterKillNewAndExistingConnectionsMode();

var startScriptCommand = new LatestStartScriptCommandBuilder().WithScriptBody(new ScriptBuilder().Print("Script Executed")).Build();

(await AssertionExtensions.Should(() => clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken)).ThrowAsync<HalibutClientException>())
.And.Message.Should().Contain("A request was sent to a polling endpoint, but the polling endpoint did not collect the request within the allowed time (00:00:05), so the request timed out.");

capabilitiesMethodUsages.ForGetCapabilitiesAsync().Completed.Should().Be(1);
scriptServiceMethodUsages.ForStartScriptAsync().Completed.Should().Be(0, "Get Capabilities should have timed out and cancelled script execution");
}

[Test]
[TentacleConfigurations(testListening: false)]
public async Task WhenThePollingRequestQueueTimeoutIsReached_ForARequestBeingProcessedByTentacle_AndTheResponseIsReceivedBeforeThePollingRequestMaximumMessageProcessingTimeoutIsReached_TheRequestShouldSucceed(TentacleConfigurationTestCase tentacleConfigurationTestCase)
{
var pollingRequestQueueTimeout = TimeSpan.FromSeconds(5);
var buffer = TimeSpan.FromSeconds(5);

await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder()
.WithPortForwarder()
.WithRetriesDisabled()
.WithServiceEndpointModifier(serviceEndpoint =>
{
serviceEndpoint.PollingRequestQueueTimeout = pollingRequestQueueTimeout;
serviceEndpoint.PollingRequestMaximumMessageProcessingTimeout = TimeSpan.FromHours(1);
})
.WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder()
.RecordMethodUsages<IAsyncClientCapabilitiesServiceV2>(out var capabilitiesMethodUsages)
.RecordMethodUsages(tentacleConfigurationTestCase, out var scriptServiceMethodUsages)
.Build())
.Build(CancellationToken);

// Ensure the script takes longer than the pollingRequestQueueTimeout but less than the PollingRequestMaximumMessageProcessingTimeout
var scriptDuration = pollingRequestQueueTimeout + buffer;

var startScriptCommand = new LatestStartScriptCommandBuilder().WithScriptBody(new ScriptBuilder()
.Sleep(scriptDuration)
.Print("Script Executed"))
// Allow the Start Script call to take longer than the script being executed
.WithDurationStartScriptCanWaitForScriptToFinish(scriptDuration * 2)
.Build();

var (finalResponse, _) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken);

finalResponse.State.Should().Be(ProcessState.Complete);
finalResponse.ExitCode.Should().Be(0);

capabilitiesMethodUsages.ForGetCapabilitiesAsync().Completed.Should().Be(1);
scriptServiceMethodUsages.ForStartScriptAsync().Completed.Should().Be(1);
scriptServiceMethodUsages.ForGetStatusAsync().Completed.Should().Be(0, "The test should have allowed StartScript long enough to complete the script, ensuring it took longer than PollingRequestQueueTimeout");
scriptServiceMethodUsages.ForCompleteScriptAsync().Completed.Should().Be(1);
}

[Test]
[TentacleConfigurations(testListening: false)]
public async Task WhenThePollingRequestMaximumMessageProcessingTimeoutIsReached_ForARequestBeingProcessedByTentacle_TheRequestShouldTimeout(TentacleConfigurationTestCase tentacleConfigurationTestCase)
{
var pollingRequestQueueTimeout = TimeSpan.FromSeconds(5);
var pollingRequestMaximumMessageProcessingTimeout = TimeSpan.FromSeconds(10);
var buffer = TimeSpan.FromSeconds(5);

await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder()
.WithPortForwarder()
.WithRetriesDisabled()
.WithServiceEndpointModifier(serviceEndpoint =>
{
serviceEndpoint.PollingRequestQueueTimeout = pollingRequestQueueTimeout;
serviceEndpoint.PollingRequestMaximumMessageProcessingTimeout = pollingRequestMaximumMessageProcessingTimeout;
})
.WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder()
.RecordMethodUsages<IAsyncClientCapabilitiesServiceV2>(out var capabilitiesMethodUsages)
.RecordMethodUsages(tentacleConfigurationTestCase, out var scriptServiceMethodUsages)
.Build())
.Build(CancellationToken);

// Ensure the script takes longer than the pollingRequestQueueTimeout + pollingRequestMaximumMessageProcessingTimeout
var scriptDuration = pollingRequestQueueTimeout + pollingRequestMaximumMessageProcessingTimeout + buffer + buffer;

var startScriptCommand = new LatestStartScriptCommandBuilder().WithScriptBody(new ScriptBuilder()
.Sleep(scriptDuration)
.Print("Script Executed"))
// Allow the Start Script call to take longer than the script being executed
.WithDurationStartScriptCanWaitForScriptToFinish(scriptDuration * 2)
.Build();

(await AssertionExtensions.Should(() => clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken)).ThrowAsync<HalibutClientException>())
.And.Message.Should().Contain("A request was sent to a polling endpoint, the polling endpoint collected it but did not respond in the allowed time (00:00:10), so the request timed out.");

capabilitiesMethodUsages.ForGetCapabilitiesAsync().Completed.Should().Be(1);
scriptServiceMethodUsages.ForStartScriptAsync().Completed.Should().Be(1);
scriptServiceMethodUsages.ForGetStatusAsync().Completed.Should().Be(0, "The test should have allowed StartScript long enough to cause the PollingRequestMaximumMessageProcessingTimeout to elapse");
scriptServiceMethodUsages.ForCancelScriptAsync().Completed.Should().Be(0);
scriptServiceMethodUsages.ForCompleteScriptAsync().Completed.Should().Be(0);
}
}
}
32 changes: 30 additions & 2 deletions source/Octopus.Tentacle.Tests.Integration/Support/TentacleConfigurationsAttribute.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,23 @@ public TentacleConfigurationsAttribute(bool testCommonVersions = false,
bool testScriptIsolationLevelVersions = false,
bool testDefaultTentacleRuntimeOnly = false,
ScriptServiceVersionToTest scriptServiceToTest = ScriptServiceVersionToTest.TentacleSupported,
bool testPolling = true,
bool testListening = true,
params object[] additionalParameterTypes)
: base(
typeof(TentacleConfigurationTestCases),
nameof(TentacleConfigurationTestCases.GetEnumerator),
new object[]
{
testCommonVersions, testCapabilitiesServiceVersions, testNoCapabilitiesServiceVersions, testScriptIsolationLevelVersions, testDefaultTentacleRuntimeOnly, scriptServiceToTest, additionalParameterTypes
testCommonVersions,
testCapabilitiesServiceVersions,
testNoCapabilitiesServiceVersions,
testScriptIsolationLevelVersions,
testDefaultTentacleRuntimeOnly,
scriptServiceToTest,
testPolling,
testListening,
additionalParameterTypes
})
{
}
Expand Down Expand Up @@ -53,9 +63,27 @@ public static IEnumerator GetEnumerator(
bool testScriptIsolationLevel,
bool testDefaultTentacleRuntimeOnly,
ScriptServiceVersionToTest scriptServiceToTest,
bool testPolling,
bool testListening,
object[] additionalParameterTypes)
{
var tentacleTypes = new[] { TentacleType.Listening, TentacleType.Polling };
var tentacleTypes = new List<TentacleType>();

if (testPolling)
{
tentacleTypes.Add(TentacleType.Polling);
}

if (testListening)
{
tentacleTypes.Add(TentacleType.Listening);
}

if (!tentacleTypes.Any())
{
throw new ArgumentException("At least one TentacleType must be tested.");
}

List<Version?> versions = new();

if (testCommonVersions)
Expand Down