Experiment

.dotnet.azure/sdk/openai/Azure.AI.OpenAI/src/Custom/RealtimeConversation/AzureRealtimeConversationClient.Protocol.cs

@@ -3,37 +3,48 @@
 
 using System.ClientModel;
 using System.ClientModel.Primitives;
+using System.ClientModel.Primitives.TwoWayClient;
 using System.ComponentModel;
 using System.Diagnostics.CodeAnalysis;
-using OpenAI.RealtimeConversation;
 
 namespace Azure.AI.OpenAI.RealtimeConversation;
 
 [Experimental("OPENAI002")]
 internal partial class AzureRealtimeConversationClient : RealtimeConversationClient
 {
-    /// <summary>
-    /// <para>[Protocol Method]</para>
-    /// Creates a new realtime conversation operation instance, establishing a connection with the /realtime endpoint.
-    /// </summary>
-    /// <param name="options"></param>
-    /// <returns></returns>
-    [EditorBrowsable(EditorBrowsableState.Never)]
-    public override async Task<RealtimeConversationSession> StartConversationSessionAsync(RequestOptions options)
+    // TODO: validate that Azure client can derive from OAI client
+    // and modify endpoint and credential as needed.
+
+    public override Task<AssistantConversation> StartConversationAsync(BinaryContent configuration, TwoWayPipelineOptions conversationOptions, RequestOptions requestOptions)
     {
-        RealtimeConversationSession provisionalOperation = _tokenCredential is not null
-            ? new AzureRealtimeConversationSession(this, _endpoint, _tokenCredential, _tokenAuthorizationScopes, _userAgent)
-            : new AzureRealtimeConversationSession(this, _endpoint, _credential, _userAgent);
-        try
-        {
-            await provisionalOperation.ConnectAsync(options).ConfigureAwait(false);
-            RealtimeConversationSession result = provisionalOperation;
-            provisionalOperation = null;
-            return result;
-        }
-        finally
-        {
-            provisionalOperation?.Dispose();
-        }
+        return base.StartConversationAsync(configuration, conversationOptions, requestOptions);
     }
+
+    ///// <summary>
+    ///// <para>[Protocol Method]</para>
+    ///// Creates a new realtime conversation operation instance, establishing a connection with the /realtime endpoint.
+    ///// </summary>
+    ///// <param name="options"></param>
+    ///// <returns></returns>
+    //[EditorBrowsable(EditorBrowsableState.Never)]
+    //public override Task<AssistantConversation> StartConversationAsync(RequestOptions options)
+    //{
+    //    throw new NotImplementedException();
+
+    //    //AssistantConversation provisionalOperation = _tokenCredential is not null
+    //    //    ? new AzureRealtimeConversationSession(this, _endpoint, _tokenCredential, _tokenAuthorizationScopes, _userAgent)
+    //    //    : new AzureRealtimeConversationSession(this, _endpoint, _credential, _userAgent);
+    //    //try
+    //    //{
+    //    //    // TODO
+    //    //    //await provisionalOperation.ConnectAsync(options).ConfigureAwait(false);
+    //    //    AssistantConversation result = provisionalOperation;
+    //    //    provisionalOperation = null;
+    //    //    return result;
+    //    //}
+    //    //finally
+    //    //{
+    //    //    provisionalOperation?.Dispose();
+    //    //}
+    //}
 }

.dotnet.azure/sdk/openai/Azure.AI.OpenAI/src/Custom/RealtimeConversation/AzureRealtimeConversationSession.Protocol.cs

@@ -1,44 +1,45 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-using Azure.Core;
-using OpenAI.RealtimeConversation;
 using System.ClientModel.Primitives;
-using System.ComponentModel;
-using System.Net.WebSockets;
+using System.ClientModel.Primitives.TwoWayClient;
 
 namespace Azure.AI.OpenAI.RealtimeConversation;
 
-internal partial class AzureRealtimeConversationSession : RealtimeConversationSession
+internal partial class AzureRealtimeConversation : AssistantConversation
 {
-    [EditorBrowsable(EditorBrowsableState.Never)]
-    protected internal override async Task ConnectAsync(RequestOptions options)
+    public AzureRealtimeConversation(PipelineResponse response, TwoWayPipelineOptions options) : base(response, options)
     {
-        ClientUriBuilder uriBuilder = new();
-        uriBuilder.Reset(_endpoint);
+    }
 
-        if (_tokenCredential is not null)
-        {
-            AccessToken token = await _tokenCredential.GetTokenAsync(_tokenRequestContext, options?.CancellationToken ?? default).ConfigureAwait(false);
-            _clientWebSocket.Options.SetRequestHeader("Authorization", $"Bearer {token.Token}");
-        }
-        else
-        {
-            _keyCredential.Deconstruct(out string dangerousCredential);
-            _clientWebSocket.Options.SetRequestHeader("api-key", dangerousCredential);
-            // uriBuilder.AppendQuery("api-key", dangerousCredential, escape: false);
-        }
+    //[EditorBrowsable(EditorBrowsableState.Never)]
+    //internal override async Task ConnectAsync(RequestOptions options)
+    //{
+    //    ClientUriBuilder uriBuilder = new();
+    //    uriBuilder.Reset(_endpoint);
 
-        Uri endpoint = uriBuilder.ToUri();
+    //    if (_tokenCredential is not null)
+    //    {
+    //        AccessToken token = await _tokenCredential.GetTokenAsync(_tokenRequestContext, options?.CancellationToken ?? default).ConfigureAwait(false);
+    //        _clientWebSocket.Options.SetRequestHeader("Authorization", $"Bearer {token.Token}");
+    //    }
+    //    else
+    //    {
+    //        _keyCredential.Deconstruct(out string dangerousCredential);
+    //        _clientWebSocket.Options.SetRequestHeader("api-key", dangerousCredential);
+    //        // uriBuilder.AppendQuery("api-key", dangerousCredential, escape: false);
+    //    }
 
-        try
-        {
-            await _clientWebSocket.ConnectAsync(endpoint, options?.CancellationToken ?? default)
-                .ConfigureAwait(false);
-        }
-        catch (WebSocketException)
-        {
-            throw;
-        }
-    }
+    //    Uri endpoint = uriBuilder.ToUri();
+
+    //    try
+    //    {
+    //        await _clientWebSocket.ConnectAsync(endpoint, options?.CancellationToken ?? default)
+    //            .ConfigureAwait(false);
+    //    }
+    //    catch (WebSocketException)
+    //    {
+    //        throw;
+    //    }
+    //}
 }

.dotnet.azure/sdk/openai/Azure.AI.OpenAI/src/Custom/RealtimeConversation/AzureRealtimeConversationSession.cs

@@ -3,6 +3,7 @@
 
 using System.ClientModel;
 using System.ClientModel.Primitives;
+using System.ClientModel.Primitives.TwoWayClient;
 using System.ComponentModel;
 using System.Diagnostics.CodeAnalysis;
 using System.Net;
@@ -12,7 +13,7 @@
 namespace Azure.AI.OpenAI.RealtimeConversation;
 
 [Experimental("OPENAI002")]
-internal partial class AzureRealtimeConversationSession : RealtimeConversationSession
+internal partial class AzureRealtimeConversationSession : AssistantConversation
 {
     private readonly Uri _endpoint;
     private readonly ApiKeyCredential _keyCredential;
@@ -44,15 +45,16 @@ protected internal AzureRealtimeConversationSession(
         _tokenRequestContext = new(_tokenAuthorizationScopes.ToArray(), parentRequestId: _clientRequestId);
     }
 
+    // TODO: make it build
     private AzureRealtimeConversationSession(AzureRealtimeConversationClient parentClient, Uri endpoint, string userAgent)
         : base(parentClient, endpoint, credential: new("placeholder"))
     {
-        _clientRequestId = Guid.NewGuid().ToString();
+        //_clientRequestId = Guid.NewGuid().ToString();
 
-        _endpoint = endpoint;
-        _clientWebSocket.Options.AddSubProtocol("realtime");
-        _clientWebSocket.Options.SetRequestHeader("User-Agent", userAgent);
-        _clientWebSocket.Options.SetRequestHeader("x-ms-client-request-id", _clientRequestId);
+        //_endpoint = endpoint;
+        //_clientWebSocket.Options.AddSubProtocol("realtime");
+        //_clientWebSocket.Options.SetRequestHeader("User-Agent", userAgent);
+        //_clientWebSocket.Options.SetRequestHeader("x-ms-client-request-id", _clientRequestId);
     }
 
     internal override async Task SendCommandAsync(InternalRealtimeRequestCommand command, CancellationToken cancellationToken = default)
@@ -67,7 +69,6 @@ internal override async Task SendCommandAsync(InternalRealtimeRequestCommand com
                 .Replace(@"""turn_detection"":null", @"""turn_detection"":{""type"":""none""}"));
         }
 
-        RequestOptions cancellationOptions = cancellationToken.ToRequestOptions();
-        await SendCommandAsync(requestData, cancellationOptions).ConfigureAwait(false);
+        await SendAsync(BinaryContent.Create(requestData), cancellationToken.ToMessageOptions()).ConfigureAwait(false);
     }
 }

.dotnet.azure/sdk/openai/Azure.AI.OpenAI/tests/ConversationProtocolTests.cs

@@ -24,7 +24,7 @@ public ConversationProtocolTests(bool isAsync) : base(isAsync)
     public async Task ProtocolCanConfigureSession()
     {
         RealtimeConversationClient client = GetTestClient();
-        using RealtimeConversationSession session = await client.StartConversationSessionAsync(CancellationToken);
+        using AssistantConversation session = await client.StartConversationAsync(CancellationToken);
 
         BinaryData configureSessionCommand = BinaryData.FromString("""
             {
@@ -38,7 +38,7 @@ public async Task ProtocolCanConfigureSession()
 
         List<JsonNode> receivedCommands = [];
 
-        await foreach (ConversationUpdate update in session.ReceiveUpdatesAsync(CancellationToken))
+        await foreach (ConversationUpdate update in session.GetResponsesAsync(CancellationToken))
         {
             BinaryData rawContentBytes = update.GetRawContent();
             JsonNode jsonNode = JsonNode.Parse(rawContentBytes);

.dotnet.azure/sdk/openai/Azure.AI.OpenAI/tests/ConversationTests.cs

@@ -24,7 +24,7 @@ public ConversationTests(bool isAsync) : base(isAsync) { }
     public async Task CanConfigureSession()
     {
         RealtimeConversationClient client = GetTestClient();
-        using RealtimeConversationSession session = await client.StartConversationSessionAsync(CancellationToken);
+        using AssistantConversation session = await client.StartConversationAsync(CancellationToken);
 
         await session.ConfigureSessionAsync(
             new ConversationSessionOptions()
@@ -39,7 +39,7 @@ await session.ConfigureSessionAsync(
 
         List<ConversationUpdate> receivedUpdates = [];
 
-        await foreach (ConversationUpdate update in session.ReceiveUpdatesAsync(CancellationToken))
+        await foreach (ConversationUpdate update in session.GetResponsesAsync(CancellationToken))
         {
             receivedUpdates.Add(update);
 
@@ -70,15 +70,15 @@ List<T> GetReceivedUpdates<T>() where T : ConversationUpdate
     public async Task TextOnlyWorks()
     {
         RealtimeConversationClient client = GetTestClient();
-        using RealtimeConversationSession session = await client.StartConversationSessionAsync(CancellationToken);
+        using AssistantConversation session = await client.StartConversationAsync(CancellationToken);
         await session.AddItemAsync(
             ConversationItem.CreateUserMessage(["Hello, world!"]),
             cancellationToken: CancellationToken);
         await session.StartResponseTurnAsync(CancellationToken);
 
         StringBuilder responseBuilder = new();
 
-        await foreach (ConversationUpdate update in session.ReceiveUpdatesAsync(CancellationToken))
+        await foreach (ConversationUpdate update in session.GetResponsesAsync(CancellationToken))
         {
             if (update is ConversationSessionStartedUpdate sessionStartedUpdate)
             {
@@ -107,7 +107,7 @@ await session.AddItemAsync(
     public async Task AudioWithToolsWorks()
     {
         RealtimeConversationClient client = GetTestClient();
-        using RealtimeConversationSession session = await client.StartConversationSessionAsync(CancellationToken);
+        using AssistantConversation session = await client.StartConversationAsync(CancellationToken);
 
         ConversationFunctionTool getWeatherTool = new()
         {
@@ -161,7 +161,7 @@ public async Task AudioWithToolsWorks()
 
         string userTranscript = null;
 
-        await foreach (ConversationUpdate update in session.ReceiveUpdatesAsync(CancellationToken))
+        await foreach (ConversationUpdate update in session.GetResponsesAsync(CancellationToken))
         {
             Assert.That(update.EventId, Is.Not.Null.And.Not.Empty);
 
@@ -211,7 +211,7 @@ public async Task AudioWithToolsWorks()
     public async Task CanDisableVoiceActivityDetection()
     {
         RealtimeConversationClient client = GetTestClient();
-        using RealtimeConversationSession session = await client.StartConversationSessionAsync(CancellationToken);
+        using AssistantConversation session = await client.StartConversationAsync(CancellationToken);
 
         await session.ConfigureSessionAsync(
             new()
@@ -231,7 +231,7 @@ await session.ConfigureSessionAsync(
 
         await session.AddItemAsync(ConversationItem.CreateUserMessage(["Hello, assistant!"]), CancellationToken);
 
-        await foreach (ConversationUpdate update in session.ReceiveUpdatesAsync(CancellationToken))
+        await foreach (ConversationUpdate update in session.GetResponsesAsync(CancellationToken))
         {
             if (update is ConversationErrorUpdate errorUpdate)
             {

.dotnet/OpenAI.sln

@@ -8,6 +8,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenAI.Examples", "examples
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenAI.Tests", "tests\OpenAI.Tests.csproj", "{6F156401-2544-41D7-B204-3148C51C1D09}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.ClientModel", "..\..\azure-sdk-for-net\sdk\core\System.ClientModel\src\System.ClientModel.csproj", "{6FA4EC28-FC48-4306-8F46-553842D4E435}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -26,6 +28,10 @@ Global
 		{6F156401-2544-41D7-B204-3148C51C1D09}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{6F156401-2544-41D7-B204-3148C51C1D09}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{6F156401-2544-41D7-B204-3148C51C1D09}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6FA4EC28-FC48-4306-8F46-553842D4E435}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6FA4EC28-FC48-4306-8F46-553842D4E435}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6FA4EC28-FC48-4306-8F46-553842D4E435}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6FA4EC28-FC48-4306-8F46-553842D4E435}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

.dotnet/api/OpenAI.netstandard2.0.cs

@@ -2210,6 +2210,25 @@ public static class OpenAIModerationsModelFactory {
     }
 }
 namespace OpenAI.RealtimeConversation {
+    public class AssistantConversation {
+        public Task AddItemAsync(ConversationItem item, string previousItemId, CancellationToken cancellationToken = default);
+        public Task AddItemAsync(ConversationItem item, CancellationToken cancellationToken = default);
+        public Task CancelResponseTurnAsync(CancellationToken cancellationToken = default);
+        public Task CommitPendingAudioAsync(CancellationToken cancellationToken = default);
+        public void CreateResponse(BinaryContent content, TwoWayClient.TwoWayMessageOptions? options = null);
+        public Task CreateResponseAsync(BinaryContent content, TwoWayClient.TwoWayMessageOptions? options = null);
+        public Task DeleteItemAsync(string itemId, CancellationToken cancellationToken = default);
+        public IEnumerable<TwoWayClient.TwoWayResult<ConversationUpdate>> GetResponses(CancellationToken cancellationToken = default);
+        public IAsyncEnumerable<TwoWayClient.TwoWayResult<ConversationUpdate>> GetResponsesAsync(CancellationToken cancellationToken = default);
+        public Task InterruptTurnAsync(CancellationToken cancellationToken = default);
+        public Task SendAudioAsync(BinaryData audio, CancellationToken cancellationToken = default);
+        public Task SendAudioAsync(Stream audio, CancellationToken cancellationToken = default);
+        public Task StartResponseTurnAsync(CancellationToken cancellationToken = default);
+    }
+    public class AssistantConversationOptions {
+        public AssistantConversationOptions();
+        public ConversationSessionOptions? ConversationOptions { get; set; }
+    }
     public class ConversationAudioDeltaUpdate : ConversationUpdate, IJsonModel<ConversationAudioDeltaUpdate>, IPersistableModel<ConversationAudioDeltaUpdate> {
         public int ContentIndex { get; }
         public BinaryData Delta { get; }
@@ -2876,39 +2895,12 @@ public class RealtimeConversationClient {
         public RealtimeConversationClient(string model, ApiKeyCredential credential, OpenAIClientOptions options);
         public RealtimeConversationClient(string model, ApiKeyCredential credential);
         public virtual ClientPipeline Pipeline { get; }
-        public event EventHandler<BinaryData> OnReceivingCommand { add; remove; }
-        public event EventHandler<BinaryData> OnSendingCommand { add; remove; }
-        public RealtimeConversationSession StartConversationSession(CancellationToken cancellationToken = default);
-        public virtual Task<RealtimeConversationSession> StartConversationSessionAsync(RequestOptions options);
-        public virtual Task<RealtimeConversationSession> StartConversationSessionAsync(CancellationToken cancellationToken = default);
-    }
-    public class RealtimeConversationSession : IDisposable {
-        protected Net.WebSockets.ClientWebSocket _clientWebSocket;
-        protected internal RealtimeConversationSession(RealtimeConversationClient parentClient, Uri endpoint, ApiKeyCredential credential);
-        public Task AddItemAsync(ConversationItem item, string previousItemId, CancellationToken cancellationToken = default);
-        public Task AddItemAsync(ConversationItem item, CancellationToken cancellationToken = default);
-        public Task CancelResponseTurnAsync(CancellationToken cancellationToken = default);
-        public Task CommitPendingAudioAsync(CancellationToken cancellationToken = default);
-        public Task ConfigureSessionAsync(ConversationSessionOptions sessionOptions, CancellationToken cancellationToken = default);
-        [EditorBrowsable(EditorBrowsableState.Never)]
-        protected internal virtual void Connect(RequestOptions options);
-        [EditorBrowsable(EditorBrowsableState.Never)]
-        protected internal virtual Task ConnectAsync(RequestOptions options);
-        public Task DeleteItemAsync(string itemId, CancellationToken cancellationToken = default);
-        public void Dispose();
-        public Task InterruptTurnAsync(CancellationToken cancellationToken = default);
-        [EditorBrowsable(EditorBrowsableState.Never)]
-        public virtual IEnumerable<ClientResult> ReceiveUpdates(RequestOptions options);
-        [EditorBrowsable(EditorBrowsableState.Never)]
-        public virtual IAsyncEnumerable<ClientResult> ReceiveUpdatesAsync(RequestOptions options);
-        public IAsyncEnumerable<ConversationUpdate> ReceiveUpdatesAsync(CancellationToken cancellationToken = default);
-        public Task SendAudioAsync(BinaryData audio, CancellationToken cancellationToken = default);
-        public Task SendAudioAsync(Stream audio, CancellationToken cancellationToken = default);
-        [EditorBrowsable(EditorBrowsableState.Never)]
-        public virtual void SendCommand(BinaryData data, RequestOptions options);
-        [EditorBrowsable(EditorBrowsableState.Never)]
-        public virtual Task SendCommandAsync(BinaryData data, RequestOptions options);
-        public Task StartResponseTurnAsync(CancellationToken cancellationToken = default);
+        public virtual AssistantConversation StartConversation(AssistantConversationOptions options, CancellationToken cancellationToken = default);
+        public virtual AssistantConversation StartConversation(BinaryContent configuration, TwoWayClient.TwoWayPipelineOptions conversationOptions, RequestOptions requestOptions);
+        public AssistantConversation StartConversation(CancellationToken cancellationToken = default);
+        public virtual Task<AssistantConversation> StartConversationAsync(AssistantConversationOptions options, CancellationToken cancellationToken = default);
+        public virtual Task<AssistantConversation> StartConversationAsync(BinaryContent configuration, TwoWayClient.TwoWayPipelineOptions conversationOptions, RequestOptions requestOptions);
+        public virtual Task<AssistantConversation> StartConversationAsync(CancellationToken cancellationToken = default);
     }
 }
 namespace OpenAI.VectorStores {