FFI v0.12.2 & RPC

.gitignore

@@ -72,3 +72,5 @@ crashlytics-build.properties
 /[Aa]ssets/[Ss]treamingAssets/aa/*
 
 .DS_Store
+
+downloads~

BuildScripts~/generate_proto.sh

@@ -14,4 +14,5 @@ protoc \
     $FFI_PROTOCOL/room.proto \
     $FFI_PROTOCOL/stats.proto \
     $FFI_PROTOCOL/track.proto \
-    $FFI_PROTOCOL/video_frame.proto
+    $FFI_PROTOCOL/video_frame.proto \
+    $FFI_PROTOCOL/rpc.proto

README.md

@@ -187,5 +187,71 @@ void TrackSubscribed(IRemoteTrack track, RemoteTrackPublication publication, Rem
 }
 ```
 
+### RPC
+
+Perform your own predefined method calls from one participant to another. 
+
+This feature is especially powerful when used with [Agents](https://docs.livekit.io/agents), for instance to forward LLM function calls to your client application.
+
+The following is a brief overview but [more detail is available in the documentation](https://docs.livekit.io/home/client/data/rpc).
+
+#### Registering an RPC method
+
+The participant who implements the method and will receive its calls must first register support. Your method handler will be an async callback that receives an `RpcInvocationData` object:
+
+```cs
+// Define your method handler
+async Task<string> HandleGreeting(RpcInvocationData data)
+{
+    Debug.Log($"Received greeting from {data.CallerIdentity}: {data.Payload}");
+    return $"Hello, {data.CallerIdentity}!";
+}
+
+// Register the method after connection to the room
+room.LocalParticipant.RegisterRpcMethod("greet", HandleGreeting);
+```
+
+In addition to the payload, `RpcInvocationData` also contains `responseTimeout`, which informs you the maximum time available to return a response. If you are unable to respond in time, the call will result in an error on the caller's side.
+
+#### Performing an RPC request
+
+The caller may initiate an RPC call using coroutines:
+
+```cs
+IEnumerator PerformRpcCoroutine()
+{
+    var rpcCall = room.LocalParticipant.PerformRpc(new PerformRpcParams
+    {
+        DestinationIdentity = "recipient-identity",
+        Method = "greet",
+        Payload = "Hello from RPC!"
+    });
+
+    yield return rpcCall;
+
+    if (rpcCall.IsError)
+    {
+        Debug.Log($"RPC call failed: {rpcCall.Error}");
+    }
+    else
+    {
+        Debug.Log($"RPC response: {rpcCall.Payload}");
+    }
+}
+
+// Start the coroutine from another MonoBehaviour method
+StartCoroutine(PerformRpcCoroutine());
+```
+
+You may find it useful to adjust the `ResponseTimeout` parameter, which indicates the amount of time you will wait for a response. We recommend keeping this value as low as possible while still satisfying the constraints of your application.
+
+#### Errors
+
+LiveKit is a dynamic realtime environment and RPC calls can fail for various reasons. 
+
+You may throw errors of the type `RpcError` with a string `message` in an RPC method handler and they will be received on the caller's side with the message intact. Other errors will not be transmitted and will instead arrive to the caller as `1500` ("Application Error"). Other built-in errors are detailed in the [docs](https://docs.livekit.io/home/client/data/rpc/#errors).
+
+
+
 <!--BEGIN_REPO_NAV-->
 <!--END_REPO_NAV-->

Runtime/Scripts/AudioSource.cs

@@ -69,7 +69,6 @@ public RtcAudioSource(AudioSource source, RtcAudioSourceType audioSourceType = R
             newAudioSource.Options.EchoCancellation = true;
             newAudioSource.Options.AutoGainControl = true;
             newAudioSource.Options.NoiseSuppression = true;
-            newAudioSource.EnableQueue = false;
             using var response = request.Send();
             FfiResponse res = response;
             _info = res.NewAudioSource.Source.Info;

Runtime/Scripts/Internal/FFIClient.cs

@@ -37,10 +37,14 @@ internal sealed class FfiClient : IFFIClient
         public event DisconnectReceivedDelegate? DisconnectReceived;
         public event RoomEventReceivedDelegate? RoomEventReceived;
         public event TrackEventReceivedDelegate? TrackEventReceived;
+        public event RpcMethodInvocationReceivedDelegate? RpcMethodInvocationReceived;
+
         // participant events are not allowed in the fii protocol public event ParticipantEventReceivedDelegate ParticipantEventReceived;
         public event VideoStreamEventReceivedDelegate? VideoStreamEventReceived;
         public event AudioStreamEventReceivedDelegate? AudioStreamEventReceived;
 
+        public event PerformRpcReceivedDelegate? PerformRpcReceived;
+
         public FfiClient() : this(Pools.NewFfiResponsePool(), new ArrayMemoryPool())
         {
         }
@@ -122,13 +126,13 @@ private static void InitializeSdk()
             const bool captureLogs = false;
 #endif
 
-            NativeMethods.LiveKitInitialize(FFICallback, captureLogs);
+            NativeMethods.LiveKitInitialize(FFICallback, captureLogs, "unity", ""); // TODO: Get SDK version
 
             Utils.Debug("FFIServer - Initialized");
             initialized = true;
         }
 
-        public void Initialize()
+        public void Initialize()    
         {
             InitializeSdk();
         }
@@ -238,6 +242,9 @@ static unsafe void FFICallback(UIntPtr data, UIntPtr size)
                     case FfiEvent.MessageOneofCase.TrackEvent:
                         Instance.TrackEventReceived?.Invoke(r.TrackEvent!);
                         break;
+                    case FfiEvent.MessageOneofCase.RpcMethodInvocation:
+                        Instance.RpcMethodInvocationReceived?.Invoke(r.RpcMethodInvocation);
+                        break;
                     case FfiEvent.MessageOneofCase.Disconnect:
                         Instance.DisconnectReceived?.Invoke(r.Disconnect!);
                         break;
@@ -251,6 +258,9 @@ static unsafe void FFICallback(UIntPtr data, UIntPtr size)
                         break;
                     case FfiEvent.MessageOneofCase.CaptureAudioFrame:
                         break;
+                    case FfiEvent.MessageOneofCase.PerformRpc:
+                        Instance.PerformRpcReceived?.Invoke(r.PerformRpc!);
+                        break;
                     case FfiEvent.MessageOneofCase.GetStats:
                     case FfiEvent.MessageOneofCase.Panic:
                         break;

Runtime/Scripts/Internal/FFIClients/FFIEvents.cs

@@ -27,10 +27,14 @@ namespace LiveKit.Internal
 
     internal delegate void TrackEventReceivedDelegate(TrackEvent e);
 
+    internal delegate void RpcMethodInvocationReceivedDelegate(RpcMethodInvocationEvent e);
+
 
     internal delegate void VideoStreamEventReceivedDelegate(VideoStreamEvent e);
 
 
     internal delegate void AudioStreamEventReceivedDelegate(AudioStreamEvent e);
 
+    internal delegate void PerformRpcReceivedDelegate(PerformRpcCallback e);
+
 }

Runtime/Scripts/Internal/FFIClients/FfiRequestExtensions.cs

@@ -84,6 +84,19 @@ public static void Inject<T>(this FfiRequest ffiRequest, T request)
                 case E2eeRequest e2EeRequest:
                     ffiRequest.E2Ee = e2EeRequest;
                     break;
+                // Rpc
+                case RegisterRpcMethodRequest registerRpcMethodRequest:
+                    ffiRequest.RegisterRpcMethod = registerRpcMethodRequest;
+                    break;
+                case UnregisterRpcMethodRequest unregisterRpcMethodRequest:
+                    ffiRequest.UnregisterRpcMethod = unregisterRpcMethodRequest;
+                    break;
+                case PerformRpcRequest performRpcRequest:
+                    ffiRequest.PerformRpc = performRpcRequest;
+                    break;
+                case RpcMethodInvocationResponseRequest rpcMethodInvocationResponseRequest:
+                    ffiRequest.RpcMethodInvocationResponse = rpcMethodInvocationResponseRequest;
+                    break;
                 default:
                     throw new Exception($"Unknown request type: {request?.GetType().FullName ?? "null"}");
             }
@@ -131,7 +144,17 @@ public static void EnsureClean(this FfiRequest request)
                 || request.CaptureAudioFrame != null
                 || request.NewAudioResampler != null
                 || request.RemixAndResample != null
+                || request.NewSoxResampler != null
+                || request.PushSoxResampler != null
+                || request.FlushSoxResampler != null
                 || request.E2Ee != null
+                ||
+
+                // Rpc
+                request.RegisterRpcMethod != null
+                || request.UnregisterRpcMethod != null
+                || request.PerformRpc != null
+                || request.RpcMethodInvocationResponse != null
             )
             {
                 throw new InvalidOperationException("Request is not cleared");
@@ -181,6 +204,16 @@ public static void EnsureClean(this FfiResponse response)
                 || response.NewAudioResampler != null
                 || response.RemixAndResample != null
                 || response.E2Ee != null
+                || response.NewSoxResampler != null
+                || response.PushSoxResampler != null
+                || response.FlushSoxResampler != null
+                ||
+
+                // Rpc
+                response.RegisterRpcMethod != null
+                || response.UnregisterRpcMethod != null
+                || response.PerformRpc != null
+                || response.RpcMethodInvocationResponse != null
             )
             {
                 throw new InvalidOperationException("Response is not cleared: ");

Runtime/Scripts/Internal/NativeMethods.cs

@@ -24,6 +24,6 @@ internal static class NativeMethods
         internal extern static unsafe FfiHandleId FfiNewRequest(byte* data, int len, out byte* dataPtr, out UIntPtr dataLen);
 
         [DllImport(Lib, CallingConvention = CallingConvention.Cdecl, EntryPoint = "livekit_ffi_initialize")]
-        internal extern static FfiHandleId LiveKitInitialize(FFICallbackDelegate cb, bool captureLogs);
+        internal extern static FfiHandleId LiveKitInitialize(FFICallbackDelegate cb, bool captureLogs, string sdk, string sdkVersion);
     }
 }