Add test for orchestrator termination

test/e2e/Apps/BasicDotNetIsolated/TerminateOrchestration.cs

@@ -0,0 +1,80 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System.Net;
+using Grpc.Core;
+using Microsoft.Azure.Functions.Worker;
+using Microsoft.Azure.Functions.Worker.Http;
+using Microsoft.DurableTask;
+using Microsoft.DurableTask.Client;
+using Microsoft.Extensions.Logging;
+
+namespace Microsoft.Azure.Durable.Tests.E2E
+{
+    public static class LongRunningOrchestration
+    {
+        [Function(nameof(LongRunningOrchestrator))]
+        public static async Task<List<string>> LongRunningOrchestrator(
+            [OrchestrationTrigger] TaskOrchestrationContext context)
+        {
+            ILogger logger = context.CreateReplaySafeLogger(nameof(HelloCities));
+            logger.LogInformation("Starting long-running orchestration.");
+            var outputs = new List<string>();
+
+            // Call our fake activity 100,000 times to simulate an orchestration that might run for >= 10,000s (2.7 hours)
+            for (int i = 0; i < 100000; i++) 
+            {
+                outputs.Add(await context.CallActivityAsync<string>(nameof(SimulatedWorkActivity), 100));
+            }
+
+            return outputs;
+        }
+
+        [Function(nameof(SimulatedWorkActivity))]
+        public static string SimulatedWorkActivity([ActivityTrigger]int sleepMs, FunctionContext executionContext)
+        {
+            // Sleep the provided number of ms to simulate a long-running activity operation
+            ILogger logger = executionContext.GetLogger("SimulatedWorkActivity");
+            logger.LogInformation("Sleeping for {sleepMs}ms.", sleepMs);
+            Thread.Sleep(sleepMs);
+            return $"Slept for {sleepMs}ms.";
+        }
+
+        [Function("LongOrchestrator_HttpStart")]
+        public static async Task<HttpResponseData> HttpStart(
+            [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req,
+            [DurableClient] DurableTaskClient client,
+            FunctionContext executionContext)
+        {
+            ILogger logger = executionContext.GetLogger("LongOrchestrator_HttpStart");
+
+            string instanceId = await client.ScheduleNewOrchestrationInstanceAsync(
+                nameof(LongRunningOrchestrator));
+
+            logger.LogInformation("Started orchestration with ID = '{instanceId}'.", instanceId);
+
+            return await client.CreateCheckStatusResponseAsync(req, instanceId);
+        }
+
+        [Function("TerminateInstance")]
+        public static async Task<HttpResponseData> Run(
+            [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req,
+            [DurableClient] DurableTaskClient client,
+            string instanceId)
+        {
+            string reason = "Long-running orchestration was terminated early.";
+            try 
+            {
+                await client.TerminateInstanceAsync(instanceId, reason);
+                return req.CreateResponse(HttpStatusCode.OK);
+            }
+            catch (RpcException ex) 
+            {
+                var response = req.CreateResponse(HttpStatusCode.BadRequest);
+                response.Headers.Add("Content-Type", "text/plain");
+                await response.WriteStringAsync(ex.Message);
+                return response;
+            }
+        }
+    }
+}

test/e2e/Tests/Helpers/DurableHelpers.cs

@@ -31,29 +31,40 @@ public OrchestrationStatusDetails(string statusQueryResponse)
         }
     }
 
-    internal static string ParseStatusQueryGetUri(HttpResponseMessage invocationStartResponse)
+    internal static async Task<string> ParseStatusQueryGetUriAsync(HttpResponseMessage invocationStartResponse)
     {
-        string? responseString = invocationStartResponse.Content?.ReadAsStringAsync().Result;
+        string? responseString = await invocationStartResponse.Content.ReadAsStringAsync();
+        return TokenizeAndGetValueFromKeyAsString(responseString, "StatusQueryGetUri");
+    }
+
+    internal static async Task<string> ParseInstanceIdAsync(HttpResponseMessage invocationStartResponse)
+    {
+        string? responseString = await invocationStartResponse.Content.ReadAsStringAsync();
+        return TokenizeAndGetValueFromKeyAsString(responseString, "Id");
+    }
+
+    internal static async Task<OrchestrationStatusDetails> GetRunningOrchestrationDetailsAsync(string statusQueryGetUri)
+    {
+        var statusQueryResponse = await _httpClient.GetAsync(statusQueryGetUri);
+
+        string? statusQueryResponseString = await statusQueryResponse.Content.ReadAsStringAsync();
+
+        return new OrchestrationStatusDetails(statusQueryResponseString);
+    }
 
-        if (string.IsNullOrEmpty(responseString))
+    private static string TokenizeAndGetValueFromKeyAsString(string? json, string key)
+    {
+        if (string.IsNullOrEmpty(json))
         {
             return string.Empty;
         }
-        JsonNode? responseJsonNode = JsonNode.Parse(responseString);
+        JsonNode? responseJsonNode = JsonNode.Parse(json);
         if (responseJsonNode == null)
         {
             return string.Empty;
         }
 
-        string? statusQueryGetUri = responseJsonNode["StatusQueryGetUri"]?.GetValue<string>();
+        string? statusQueryGetUri = responseJsonNode[key]?.GetValue<string>();
         return statusQueryGetUri ?? string.Empty;
     }
-    internal static OrchestrationStatusDetails GetRunningOrchestrationDetails(string statusQueryGetUri)
-    {
-        var statusQueryResponse = _httpClient.GetAsync(statusQueryGetUri);
-
-        string? statusQueryResponseString = statusQueryResponse.Result.Content.ReadAsStringAsync().Result;
-
-        return new OrchestrationStatusDetails(statusQueryResponseString);
-    }
 }

test/e2e/Tests/Tests/HelloCitiesTest.cs

@@ -42,9 +42,9 @@ public async Task HttpTriggerTests(string functionName, HttpStatusCode expectedS
         string actualMessage = await response.Content.ReadAsStringAsync();
 
         Assert.Equal(expectedStatusCode, response.StatusCode);
-        string statusQueryGetUri = DurableHelpers.ParseStatusQueryGetUri(response);
+        string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(response);
         Thread.Sleep(1000);
-        var orchestrationDetails = DurableHelpers.GetRunningOrchestrationDetails(statusQueryGetUri);
+        var orchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri);
         Assert.Equal("Completed", orchestrationDetails.RuntimeStatus);
         Assert.Contains(partialExpectedOutput, orchestrationDetails.Output);
     }
@@ -61,28 +61,28 @@ public async Task ScheduledStartTests(string functionName, int startDelaySeconds
         using HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger(functionName, urlQueryString);
         string actualMessage = await response.Content.ReadAsStringAsync();
 
-        string statusQueryGetUri = DurableHelpers.ParseStatusQueryGetUri(response);
+        string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(response);
 
         Assert.Equal(expectedStatusCode, response.StatusCode);
 
-        var orchestrationDetails = DurableHelpers.GetRunningOrchestrationDetails(statusQueryGetUri);
+        var orchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri);
         while (DateTime.UtcNow < scheduledStartTime + TimeSpan.FromSeconds(-1))
         {
             WriteOutput($"Test scheduled for {scheduledStartTime}, current time {DateTime.Now}");
-            orchestrationDetails = DurableHelpers.GetRunningOrchestrationDetails(statusQueryGetUri);
+            orchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri);
             Assert.Equal("Pending", orchestrationDetails.RuntimeStatus);
             Thread.Sleep(1000);
         }
 
         // Give a small amount of time for the orchestration to complete, even if scheduled to run immediately
         Thread.Sleep(3000);
         WriteOutput($"Test scheduled for {scheduledStartTime}, current time {DateTime.Now}, looking for completed");
-        var finalOrchestrationDetails = DurableHelpers.GetRunningOrchestrationDetails(statusQueryGetUri);
+        var finalOrchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri);
         int retryAttempts = 0;
         while (finalOrchestrationDetails.RuntimeStatus != "Completed" && retryAttempts < 10)
         {
             Thread.Sleep(1000);
-            finalOrchestrationDetails = DurableHelpers.GetRunningOrchestrationDetails(statusQueryGetUri);
+            finalOrchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri);
             retryAttempts++;
         }
         Assert.Equal("Completed", finalOrchestrationDetails.RuntimeStatus);

test/e2e/Tests/Tests/TerminateOrchestratorTests.cs

@@ -0,0 +1,154 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System.Net;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Microsoft.Azure.Durable.Tests.DotnetIsolatedE2E;
+
+[Collection(Constants.FunctionAppCollectionName)]
+public class TerminateOrchestratorTests
+{
+    private readonly FunctionAppFixture _fixture;
+    private readonly ITestOutputHelper _output;
+
+    public TerminateOrchestratorTests(FunctionAppFixture fixture, ITestOutputHelper testOutputHelper)
+    {
+        _fixture = fixture;
+        _fixture.TestLogs.UseTestLogger(testOutputHelper);
+        _output = testOutputHelper;
+    }
+
+
+    [Fact]
+    public async Task TerminateRunningOrchestration_ShouldSucceed()
+    {
+        using HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger("LongOrchestrator_HttpStart", "");
+
+        Assert.Equal(HttpStatusCode.Accepted, response.StatusCode);
+        string instanceId = await DurableHelpers.ParseInstanceIdAsync(response);
+        string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(response);
+
+        Thread.Sleep(1000);
+
+        var orchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri);
+        Assert.Equal("Running", orchestrationDetails.RuntimeStatus);
+
+        using HttpResponseMessage terminateResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={instanceId}");
+        await AssertTerminateRequestSucceedsAsync(terminateResponse);
+
+        Thread.Sleep(1000);
+
+        orchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri);
+        Assert.Equal("Terminated", orchestrationDetails.RuntimeStatus);
+    }
+
+
+    [Fact(Skip = "Will enable when https://github.com/Azure/azure-functions-durable-extension/issues/3025 is fixed")]
+    public async Task TerminateScheduledOrchestration_ShouldSucceed()
+    {
+        DateTime scheduledStartTime = DateTime.UtcNow + TimeSpan.FromMinutes(1);
+        using HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger("HelloCities_HttpStart_Scheduled", $"?scheduledStartTime={scheduledStartTime.ToString("o")}");
+
+        Assert.Equal(HttpStatusCode.Accepted, response.StatusCode);
+        string instanceId = await DurableHelpers.ParseInstanceIdAsync(response);
+        string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(response);
+
+        Thread.Sleep(1000);
+
+        var orchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri);
+        Assert.Equal("Pending", orchestrationDetails.RuntimeStatus);
+
+        using HttpResponseMessage terminateResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={instanceId}");
+        await AssertTerminateRequestSucceedsAsync(terminateResponse);
+
+        Thread.Sleep(1000);
+
+        orchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri);
+        Assert.Equal("Terminated", orchestrationDetails.RuntimeStatus);
+    }
+
+
+    [Fact]
+    public async Task TerminateTerminatedOrchestration_ShouldFail()
+    {
+        using HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger("LongOrchestrator_HttpStart", "");
+
+        Assert.Equal(HttpStatusCode.Accepted, response.StatusCode);
+        string instanceId = await DurableHelpers.ParseInstanceIdAsync(response);
+        string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(response);
+
+        Thread.Sleep(1000);
+
+        var orchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri);
+        Assert.Equal("Running", orchestrationDetails.RuntimeStatus);
+
+        using HttpResponseMessage terminateResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={instanceId}");
+        await AssertTerminateRequestSucceedsAsync(terminateResponse);
+
+        Thread.Sleep(1000);
+        using HttpResponseMessage terminateAgainResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={instanceId}");
+        await AssertTerminateRequestFailsAsync(terminateAgainResponse);
+
+        // Give some time for Core Tools to write logs out
+        Thread.Sleep(500);
+
+        Assert.Contains(_fixture.TestLogs.CoreToolsLogs, x => x.Contains("Cannot terminate orchestration instance in the Terminated state.") &&
+                                                              x.Contains(instanceId));
+
+        orchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri);
+        Assert.Equal("Terminated", orchestrationDetails.RuntimeStatus);
+    }
+
+
+    [Fact]
+    public async Task TerminateCompletedOrchestration_ShouldFail()
+    {
+        using HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger("HelloCities_HttpStart", "");
+
+        Assert.Equal(HttpStatusCode.Accepted, response.StatusCode);
+        string instanceId = await DurableHelpers.ParseInstanceIdAsync(response);
+        string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(response);
+
+        Thread.Sleep(1000);
+
+        var orchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri);
+        Assert.Equal("Completed", orchestrationDetails.RuntimeStatus);
+
+        using HttpResponseMessage terminateResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={instanceId}");
+        await AssertTerminateRequestFailsAsync(terminateResponse);
+
+        // Give some time for Core Tools to write logs out
+        Thread.Sleep(500);
+
+        Assert.Contains(_fixture.TestLogs.CoreToolsLogs, x => x.Contains("Cannot terminate orchestration instance in the Completed state.") &&
+                                                              x.Contains(instanceId));
+    }
+
+    [Fact]
+    public async Task TerminateNonExistantOrchestration_ShouldFail()
+    {
+        using HttpResponseMessage terminateResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={Guid.NewGuid().ToString()}");
+        await AssertTerminateRequestFailsAsync(terminateResponse);
+    }
+
+    private static async Task AssertTerminateRequestFailsAsync(HttpResponseMessage terminateResponse)
+    {
+        Assert.Equal(HttpStatusCode.BadRequest, terminateResponse.StatusCode);
+
+        string? terminateResponseMessage = await terminateResponse.Content.ReadAsStringAsync();
+        Assert.NotNull(terminateResponseMessage);
+        // Unclear error message - see https://github.com/Azure/azure-functions-durable-extension/issues/3027, will update this code when that bug is fixed
+        Assert.Equal("Status(StatusCode=\"Unknown\", Detail=\"Exception was thrown by handler.\")", terminateResponseMessage);
+    }
+
+    private static async Task AssertTerminateRequestSucceedsAsync(HttpResponseMessage terminateResponse)
+    {
+        Assert.Equal(HttpStatusCode.OK, terminateResponse.StatusCode);
+
+        string? terminateResponseMessage = await terminateResponse.Content.ReadAsStringAsync();
+        Assert.NotNull(terminateResponseMessage);
+        Assert.Empty(terminateResponseMessage);
+    }
+}