.Net: [FeatureBranch][FlowPlanner] Support referenced flow step (micr…

…osoft#2920)

### Motivation and Context
Note that this PR is for feature branch only.

Changes
1. Support optional completion type
2. Support reference flow as a FlowStep
3. Accommodate latest kernel changes
4. Add more unit tests for the Planner 

### Description
as above

### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [X] The code builds clean without any errors or warnings
- [X] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [X] All unit tests pass, and I have added new tests where possible
- [X] I didn't break anyone 😄

dotnet/samples/KernelSyntaxExamples/Example57_FlowPlanner.cs

@@ -14,10 +14,10 @@
 using Microsoft.SemanticKernel.Orchestration;
 using Microsoft.SemanticKernel.Planning;
 using Microsoft.SemanticKernel.Planning.Flow;
+using Microsoft.SemanticKernel.Plugins.Core;
+using Microsoft.SemanticKernel.Plugins.Web;
+using Microsoft.SemanticKernel.Plugins.Web.Bing;
 using Microsoft.SemanticKernel.SkillDefinition;
-using Microsoft.SemanticKernel.Skills.Core;
-using Microsoft.SemanticKernel.Skills.Web;
-using Microsoft.SemanticKernel.Skills.Web.Bing;
 
 /**
  * This example shows how to use FlowPlanner to execute a given flow with interaction with client.
@@ -63,11 +63,11 @@ public static Task RunAsync()
     public static async Task RunInteractiveAsync()
     {
         var bingConnector = new BingConnector(TestConfiguration.Bing.ApiKey);
-        var webSearchEngineSkill = new WebSearchEngineSkill(bingConnector);
+        var webSearchEngineSkill = new WebSearchEnginePlugin(bingConnector);
         Dictionary<object, string?> skills = new()
         {
             { webSearchEngineSkill, "WebSearch" },
-            { new TimeSkill(), "time" }
+            { new TimePlugin(), "time" }
         };
 
         FlowPlanner planner = new(GetKernelBuilder(), new FlowStatusProvider(new VolatileMemoryStore()), skills);
@@ -108,11 +108,11 @@ public static async Task RunInteractiveAsync()
     private static async Task RunExampleAsync()
     {
         var bingConnector = new BingConnector(TestConfiguration.Bing.ApiKey);
-        var webSearchEngineSkill = new WebSearchEngineSkill(bingConnector);
+        var webSearchEngineSkill = new WebSearchEnginePlugin(bingConnector);
         Dictionary<object, string?> skills = new()
         {
             { webSearchEngineSkill, "WebSearch" },
-            { new TimeSkill(), "time" }
+            { new TimePlugin(), "time" }
         };
 
         FlowPlanner planner = new(GetKernelBuilder(), new FlowStatusProvider(new VolatileMemoryStore()), skills);
@@ -166,7 +166,7 @@ private static KernelBuilder GetKernelBuilder()
             });
     }
 
-    public sealed class CollectEmailSkill : ChatSkill
+    public sealed class CollectEmailSkill
     {
         private const string Goal = "Prompt user to provide a valid email address";
 
@@ -206,7 +206,7 @@ public async Task<string> CollectEmailAsync(
             var chat = this._chat.CreateNewChat(SystemPrompt);
             chat.AddUserMessage(Goal);
 
-            ChatHistory? chatHistory = this.GetChatHistory(context);
+            ChatHistory? chatHistory = context.GetChatHistory();
             if (chatHistory?.Any() ?? false)
             {
                 chat.Messages.AddRange(chatHistory);
@@ -220,7 +220,7 @@ public async Task<string> CollectEmailAsync(
             }
 
             context.Variables["email_address"] = string.Empty;
-            this.PromptInput(context);
+            context.PromptInput();
 
             return await this._chat.GenerateMessageAsync(chat, this._chatRequestSettings).ConfigureAwait(false);
         }

dotnet/src/Extensions/Extensions.UnitTests/Planning/FlowPlanner/FlowExtensionsTests.cs

@@ -0,0 +1,122 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace SemanticKernel.Extensions.UnitTests.Planning.FlowPlanner;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.SemanticKernel.Planning.Flow;
+using Xunit;
+
+public class FlowExtensionsTests
+{
+    [Fact]
+    public async Task TestBuildReferenceStep()
+    {
+        // Arrange
+        var flow1 = CreateFlowWithReferenceStep("flow2");
+
+        var flow2 = new Flow("flow2", "test flow goal 2");
+        flow2.CompletionType = CompletionType.Optional;
+        var step5 = new FlowStep("step1");
+        step5.AddRequires("a");
+        step5.AddProvides("b");
+        flow2.AddProvides("b");
+        flow2.AddStep(step5);
+
+        // Act
+        var catalog = new InMemoryFlowCatalog(new List<Flow> { flow1, flow2 });
+        var flow1InCatalog = await catalog.GetFlowAsync("flow1").ConfigureAwait(false);
+        Assert.NotNull(flow1InCatalog);
+
+        // Assert
+        Assert.DoesNotContain(flow1InCatalog.Steps, step => step is ReferenceFlowStep);
+        var flow2Step = flow1InCatalog.Steps.OfType<Flow>().SingleOrDefault();
+        Assert.NotNull(flow2Step);
+        Assert.Equal("flow2", flow2Step.Name);
+        Assert.Equal(CompletionType.Optional, flow2Step.CompletionType);
+        Assert.Equal("a", flow2Step.Requires.SingleOrDefault());
+        Assert.Equal("b", flow2Step.Provides.SingleOrDefault());
+    }
+
+    [Fact]
+    public void TestBuildNonExistReferenceStep()
+    {
+        // Arrange
+        var flow1 = CreateFlowWithReferenceStep("flow2");
+
+        var flow2 = new Flow("flow3", "test flow goal 2");
+        var step5 = new FlowStep("step1");
+        step5.AddProvides("a");
+        flow2.AddProvides("a");
+        flow2.AddStep(step5);
+
+        // Act and assert
+        Assert.Throws<AggregateException>(() => new InMemoryFlowCatalog(new List<Flow> { flow1, flow2 }));
+    }
+
+    private static Flow CreateFlowWithReferenceStep(string referenceFlowName)
+    {
+        var flow = new Flow("flow1", "test flow goal");
+        var step1 = new FlowStep("step1");
+        step1.AddProvides("a");
+        var step2 = new FlowStep("step2");
+        step2.AddRequires("a");
+        step2.AddProvides("b");
+        var step3 = new FlowStep("step3");
+        step3.AddRequires("a", "b");
+        step3.AddProvides("c");
+        var step4 = new ReferenceFlowStep(referenceFlowName)
+        {
+            CompletionType = CompletionType.Optional
+        };
+        flow.AddStep(step1);
+        flow.AddStep(step2);
+        flow.AddStep(step3);
+        flow.AddStep(step4);
+
+        return flow;
+    }
+
+    private sealed class InMemoryFlowCatalog : IFlowCatalog
+    {
+        private readonly Dictionary<string, Flow> _flows = new();
+
+        internal InMemoryFlowCatalog()
+        {
+        }
+
+        internal InMemoryFlowCatalog(IReadOnlyList<Flow> flows)
+        {
+            // phase 1: register original flows
+            foreach (var flow in flows)
+            {
+                this._flows.Add(flow.Name, flow);
+            }
+
+            // phase 2: build references
+            foreach (var flow in flows)
+            {
+                flow.BuildReferenceAsync(this).Wait();
+            }
+        }
+
+        public Task<IEnumerable<Flow>> GetFlowsAsync()
+        {
+            return Task.FromResult(this._flows.Select(_ => _.Value));
+        }
+
+        public Task<Flow?> GetFlowAsync(string flowName)
+        {
+            return Task.FromResult(this._flows.TryGetValue(flowName, out var flow) ? flow : null);
+        }
+
+        public Task<bool> RegisterFlowAsync(Flow flow)
+        {
+            this._flows.Add(flow.Name, flow);
+
+            return Task.FromResult(true);
+        }
+    }
+}

dotnet/src/Extensions/Extensions.UnitTests/Planning/FlowPlanner/FlowSerializerTests.cs

@@ -42,16 +42,23 @@ private void ValidateFlow(Flow? flow)
         Assert.NotNull(flow);
         Assert.NotEmpty(flow.Steps);
         Assert.False(string.IsNullOrEmpty(flow.Goal));
-        Assert.Equal("breakfast", flow.Provides.Single());
-        Assert.Equal(4, flow.Steps.Count);
+        Assert.Contains("breakfast", flow.Provides);
+        Assert.Equal(5, flow.Steps.Count);
 
         var makeCoffeeStep = flow.Steps.First(step => step.Goal == "Make coffee");
         Assert.Equal("coffee_bean", makeCoffeeStep.Requires.Single());
         Assert.Equal("coffee", makeCoffeeStep.Provides.Single());
+        Assert.NotNull(makeCoffeeStep.Skills);
+        Assert.Single(makeCoffeeStep.Skills);
         Assert.Equal(CompletionType.Once, makeCoffeeStep.CompletionType);
 
         var recipeStep = flow.Steps.First(step => step.Goal == "Recipe");
         Assert.Equal("ingredients", recipeStep.Provides.Single());
         Assert.Equal(CompletionType.AtLeastOnce, recipeStep.CompletionType);
+
+        var lunchStep = flow.Steps.First(step => step is ReferenceFlowStep) as ReferenceFlowStep;
+        Assert.NotNull(lunchStep);
+        Assert.Equal(CompletionType.Optional, lunchStep.CompletionType);
+        Assert.Equal("lunch_flow", lunchStep.FlowName);
     }
 }

dotnet/src/Extensions/Extensions.UnitTests/Planning/FlowPlanner/FlowValidatorTests.cs

@@ -0,0 +1,113 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace SemanticKernel.Extensions.UnitTests.Planning.FlowPlanner;
+
+using System;
+using Microsoft.SemanticKernel.Planning.Flow;
+using Xunit;
+
+public class FlowValidatorTests
+{
+    [Fact]
+    public void TestValidateFlowReturnsTrueForValidFlow()
+    {
+        // Arrange
+        var validator = new FlowValidator();
+        var flow = new Flow("test_flow", "test flow goal");
+        var step1 = new FlowStep("step1");
+        step1.AddProvides("a");
+        var step2 = new FlowStep("step2");
+        step2.AddRequires("a");
+        step2.AddProvides("b");
+        var step3 = new FlowStep("step3");
+        step3.AddRequires("a", "b");
+        step3.AddProvides("c");
+        var step4 = new ReferenceFlowStep("another flow")
+        {
+            CompletionType = CompletionType.Optional,
+            StartingMessage = "Would you like to start another flow?"
+        };
+        flow.AddStep(step1);
+        flow.AddStep(step2);
+        flow.AddStep(step3);
+        flow.AddStep(step4);
+
+        // Act and assert
+        validator.Validate(flow);
+    }
+
+    [Fact]
+    public void TestValidateFlowThrowForEmptyFlow()
+    {
+        // Arrange
+        var validator = new FlowValidator();
+        var flow = new Flow("empty flow", "empty flow");
+
+        // Act and assert
+        Assert.Throws<ArgumentException>(() => validator.Validate(flow));
+    }
+
+    [Fact]
+    public void TestValidateFlowThrowForFlowWithDependencyLoops()
+    {
+        // Arrange
+        var validator = new FlowValidator();
+        var flow = new Flow("test_flow", "test flow goal");
+        var step1 = new FlowStep("step1");
+        step1.AddRequires("a");
+        step1.AddProvides("b");
+        var step2 = new FlowStep("step2");
+        step2.AddRequires("b");
+        step2.AddProvides("a");
+        flow.AddStep(step1);
+        flow.AddStep(step2);
+
+        // Act and assert
+        Assert.Throws<ArgumentException>(() => validator.Validate(flow));
+    }
+
+    [Fact]
+    public void TestValidateFlowThrowForReferenceStepWithRequires()
+    {
+        // Arrange
+        var validator = new FlowValidator();
+        var flow = new Flow("test_flow", "test flow goal");
+        var step1 = new ReferenceFlowStep("another flow");
+        step1.AddRequires("a");
+
+        // Act and assert
+        Assert.Throws<ArgumentException>(() => validator.Validate(flow));
+    }
+
+    [Fact]
+    public void TestValidateFlowThrowForReferenceStepWithProvides()
+    {
+        // Arrange
+        var validator = new FlowValidator();
+        var flow = new Flow("test_flow", "test flow goal");
+        var step1 = new ReferenceFlowStep("another flow");
+        step1.AddProvides("a");
+
+        // Act and assert
+        Assert.Throws<ArgumentException>(() => validator.Validate(flow));
+    }
+
+    [Fact]
+    public void TestValidateFlowThrowForOptionalStepWithoutStartingMessage()
+    {
+        // Arrange
+        var validator = new FlowValidator();
+        var flow = new Flow("test_flow", "test flow goal");
+        var step1 = new FlowStep("step1");
+        step1.AddProvides("a");
+        var step2 = new ReferenceFlowStep("another flow")
+        {
+            CompletionType = CompletionType.Optional
+        };
+        flow.AddStep(step1);
+        flow.AddStep(step2);
+
+        // Act and assert
+        Assert.Throws<ArgumentException>(() => validator.Validate(flow));
+    }
+}