Load focus information from well known and use client config only as …

…a fallback. (#2358)

* Load focus information from well known and use client config only as a fallback.

Signed-off-by: Timo K <[email protected]>
Co-authored-by: Andrew Ferrazzutti <[email protected]>

README.md

@@ -64,6 +64,28 @@ experimental_features:
 MSC3266 allows to request a room summary of rooms you are not joined.
 The summary contains the room join rules. We need that to decide if the user gets prompted with the option to knock ("ask to join"), a cannot join error or the join view.
 
+Element Call requires a Livekit SFU behind a Livekit jwt service to work. The url to the Livekit jwt service can either be configured in the config of Element Call (fallback/legacy configuration) or be configured by your homeserver via the `.well-known`.
+This is the recommended method.
+
+The configuration is a list of Foci configs:
+
+```json
+"org.matrix.msc4143.rtc_foci": [
+    {
+        "type": "livekit",
+        "livekit_service_url": "https://someurl.com"
+    },
+     {
+        "type": "livekit",
+        "livekit_service_url": "https://livekit2.com"
+    },
+    {
+        "type": "another_foci",
+        "props_for_another_foci": "val"
+    },
+]
+```
+
 ## Translation
 
 If you'd like to help translate Element Call, head over to [Localazy](https://localazy.com/p/element-call). You're also encouraged to join the [Element Translators](https://matrix.to/#/#translators:element.io) space to discuss and coordinate translation efforts.
@@ -103,7 +125,9 @@ service for development. These use a test 'secret' published in this
 repository, so this must be used only for local development and
 **_never be exposed to the public Internet._**
 
-To use it, add SFU parameter in your local config `./public/config.json`:
+To use it, add a SFU parameter in your local config `./public/config.json`:
+(Be aware, that this is only the fallback Livekit SFU. If the homeserver
+advertises one in the client well-known, this will not be used.)
 
 ```json
 "livekit": {

package.json

@@ -63,7 +63,7 @@
     "i18next-http-backend": "^2.0.0",
     "livekit-client": "^2.0.2",
     "lodash": "^4.17.21",
-    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#e874468ba3e84819cf4b342d2e66af67ab4cf804",
+    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#d754392410a23526ce65fd2fd6b04bfac989b666",
     "matrix-widget-api": "^1.3.1",
     "normalize.css": "^8.0.1",
     "pako": "^2.0.4",

src/ClientContext.tsx

@@ -25,7 +25,11 @@ import {
   useMemo,
 } from "react";
 import { useHistory } from "react-router-dom";
-import { ClientEvent, MatrixClient } from "matrix-js-sdk/src/client";
+import {
+  ClientEvent,
+  ICreateClientOpts,
+  MatrixClient,
+} from "matrix-js-sdk/src/client";
 import { logger } from "matrix-js-sdk/src/logger";
 import { useTranslation } from "react-i18next";
 import { ISyncStateData, SyncState } from "matrix-js-sdk/src/sync";
@@ -360,13 +364,13 @@ async function loadClient(): Promise<InitResult | null> {
 
       /* eslint-disable camelcase */
       const { user_id, device_id, access_token, passwordlessUser } = session;
-      const initClientParams = {
+      const initClientParams: ICreateClientOpts = {
         baseUrl: Config.defaultHomeserverUrl()!,
         accessToken: access_token,
         userId: user_id,
         deviceId: device_id,
         fallbackICEServerAllowed: fallbackICEServerAllowed,
-        livekitServiceURL: Config.get().livekit!.livekit_service_url,
+        livekitServiceURL: Config.get().livekit?.livekit_service_url,
       };
 
       try {

src/config/ConfigOptions.ts

@@ -55,7 +55,12 @@ export interface ConfigOptions {
 
   // Describes the LiveKit configuration to be used.
   livekit?: {
-    // The link to the service that returns a livekit url and token to use it
+    // The link to the service that returns a livekit url and token to use it.
+    // This is a fallback link in case the homeserver in use does not advertise
+    // a livekit service url in the client well-known.
+    // The well known needs to be formatted like so:
+    // {"type":"livekit", "livekit_service_url":"https://livekit.example.com"}
+    // and stored under the key: "livekit_focus"
     livekit_service_url: string;
   };
 

src/livekit/LivekitFocus.ts

@@ -1,23 +0,0 @@
-/*
-Copyright 2023 New Vector Ltd
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-import { Focus } from "matrix-js-sdk/src/matrixrtc/focus";
-
-export interface LivekitFocus extends Focus {
-  type: "livekit";
-  livekit_service_url: string;
-  livekit_alias: string;
-}

src/livekit/openIDSFU.ts

@@ -18,9 +18,9 @@ import { IOpenIDToken, MatrixClient } from "matrix-js-sdk";
 import { logger } from "matrix-js-sdk/src/logger";
 import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
 import { useEffect, useState } from "react";
+import { LivekitFocus } from "matrix-js-sdk/src/matrixrtc/LivekitFocus";
 
-import { LivekitFocus } from "./LivekitFocus";
-import { useActiveFocus } from "../room/useActiveFocus";
+import { useActiveLivekitFocus } from "../room/useActiveFocus";
 
 export interface SFUConfig {
   url: string;
@@ -46,7 +46,7 @@ export function useOpenIDSFU(
 ): SFUConfig | undefined {
   const [sfuConfig, setSFUConfig] = useState<SFUConfig | undefined>(undefined);
 
-  const activeFocus = useActiveFocus(rtcSession);
+  const activeFocus = useActiveLivekitFocus(rtcSession);
 
   useEffect(() => {
     (async (): Promise<void> => {

src/matrix-utils.ts

@@ -206,7 +206,7 @@ export async function initClient(
   // Otherwise, a sync may complete before the listener gets applied,
   // and we will miss it.
   const syncPromise = waitForSync(client);
-  await client.startClient();
+  await client.startClient({ clientWellKnownPollPeriod: 60 * 10 });
   await syncPromise;
 
   return client;

src/room/GroupCallView.tsx

@@ -196,7 +196,7 @@ export const GroupCallView: FC<Props> = ({
         ev: CustomEvent<IWidgetApiRequest>,
       ): Promise<void> => {
         defaultDeviceSetup(ev.detail.data as unknown as JoinCallData);
-        enterRTCSession(rtcSession, perParticipantE2EE);
+        await enterRTCSession(rtcSession, perParticipantE2EE);
         await widget!.api.transport.reply(ev.detail, {});
       };
       widget.lazyActions.on(ElementWidgetActions.JoinCall, onJoin);
@@ -318,7 +318,7 @@ export const GroupCallView: FC<Props> = ({
         client={client}
         matrixInfo={matrixInfo}
         muteStates={muteStates}
-        onEnter={(): void => enterRTCSession(rtcSession, perParticipantE2EE)}
+        onEnter={() => void enterRTCSession(rtcSession, perParticipantE2EE)}
         confineToRoom={confineToRoom}
         hideHeader={hideHeader}
         participantCount={participantCount}

src/room/useActiveFocus.ts

@@ -21,33 +21,28 @@ import {
 import { useCallback, useEffect, useState } from "react";
 import { deepCompare } from "matrix-js-sdk/src/utils";
 import { logger } from "matrix-js-sdk/src/logger";
-
-import { LivekitFocus } from "../livekit/LivekitFocus";
-
-function getActiveFocus(
-  rtcSession: MatrixRTCSession,
-): LivekitFocus | undefined {
-  const oldestMembership = rtcSession.getOldestMembership();
-  const focus = oldestMembership?.getActiveFoci()[0] as LivekitFocus;
-
-  return focus;
-}
+import {
+  LivekitFocus,
+  isLivekitFocus,
+} from "matrix-js-sdk/src/matrixrtc/LivekitFocus";
 
 /**
  * Gets the currently active (livekit) focus for a MatrixRTC session
  * This logic is specific to livekit foci where the whole call must use one
  * and the same focus.
  */
-export function useActiveFocus(
+export function useActiveLivekitFocus(
   rtcSession: MatrixRTCSession,
 ): LivekitFocus | undefined {
-  const [activeFocus, setActiveFocus] = useState(() =>
-    getActiveFocus(rtcSession),
-  );
+  const [activeFocus, setActiveFocus] = useState(() => {
+    const f = rtcSession.getActiveFocus();
+    // Only handle foci with type="livekit" for now.
+    return !!f && isLivekitFocus(f) ? f : undefined;
+  });
 
   const onMembershipsChanged = useCallback(() => {
-    const newActiveFocus = getActiveFocus(rtcSession);
-
+    const newActiveFocus = rtcSession.getActiveFocus();
+    if (!!newActiveFocus && !isLivekitFocus(newActiveFocus)) return;
     if (!deepCompare(activeFocus, newActiveFocus)) {
       const oldestMembership = rtcSession.getOldestMembership();
       logger.warn(

src/room/useLoadGroupCall.ts

@@ -217,15 +217,15 @@ export const useLoadGroupCall = (
             "Room not found. The widget-api did not pass over the relevant room events/information.",
           );
 
-        // If the room does not exist we first search for it with viaServers
-        const roomSummary = await client.getRoomSummary(roomId, viaServers);
         if (membership === KnownMembership.Ban) {
           throw bannedError();
         } else if (membership === KnownMembership.Invite) {
           room = await client.joinRoom(roomId, {
             viaServers,
           });
         } else {
+          // If the room does not exist we first search for it with viaServers
+          const roomSummary = await client.getRoomSummary(roomId, viaServers);
           if (roomSummary.join_rule === JoinRule.Public) {
             room = await client.joinRoom(roomSummary.room_id, {
               viaServers,

src/rtcSessionHelpers.ts

@@ -15,29 +15,90 @@ limitations under the License.
 */
 
 import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
+import { logger } from "matrix-js-sdk/src/logger";
+import {
+  LivekitFocus,
+  LivekitFocusActive,
+  isLivekitFocus,
+  isLivekitFocusConfig,
+} from "matrix-js-sdk/src/matrixrtc/LivekitFocus";
 
 import { PosthogAnalytics } from "./analytics/PosthogAnalytics";
-import { LivekitFocus } from "./livekit/LivekitFocus";
 import { Config } from "./config/Config";
 import { ElementWidgetActions, WidgetHelpers, widget } from "./widget";
 
-function makeFocus(livekitAlias: string): LivekitFocus {
-  const urlFromConf = Config.get().livekit!.livekit_service_url;
-  if (!urlFromConf) {
-    throw new Error("No livekit_service_url is configured!");
-  }
+const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci";
 
+export function makeActiveFocus(): LivekitFocusActive {
   return {
     type: "livekit",
-    livekit_service_url: urlFromConf,
-    livekit_alias: livekitAlias,
+    focus_selection: "oldest_membership",
   };
 }
 
-export function enterRTCSession(
+async function makePreferredLivekitFoci(
+  rtcSession: MatrixRTCSession,
+  livekitAlias: string,
+): Promise<LivekitFocus[]> {
+  logger.log("Start building foci_preferred list: ", rtcSession.room.roomId);
+
+  const preferredFoci: LivekitFocus[] = [];
+
+  // Make the Focus from the running rtc session the highest priority one
+  // This minimizes how often we need to switch foci during a call.
+  const focusInUse = rtcSession.getFocusInUse();
+  if (focusInUse && isLivekitFocus(focusInUse)) {
+    logger.log("Adding livekit focus from oldest member: ", focusInUse);
+    preferredFoci.push(focusInUse);
+  }
+
+  // Prioritize the client well known over the configured sfu.
+  const wellKnownFoci =
+    rtcSession.room.client.getClientWellKnown()?.[FOCI_WK_KEY];
+  if (Array.isArray(wellKnownFoci)) {
+    preferredFoci.push(
+      ...wellKnownFoci
+        .filter((f) => !!f)
+        .filter(isLivekitFocusConfig)
+        .map((wellKnownFocus) => {
+          logger.log("Adding livekit focus from well known: ", wellKnownFocus);
+          return { ...wellKnownFocus, livekit_alias: livekitAlias };
+        }),
+    );
+  }
+
+  const urlFromConf = Config.get().livekit?.livekit_service_url;
+  if (urlFromConf) {
+    const focusFormConf: LivekitFocus = {
+      type: "livekit",
+      livekit_service_url: urlFromConf,
+      livekit_alias: livekitAlias,
+    };
+    logger.log("Adding livekit focus from config: ", focusFormConf);
+    preferredFoci.push(focusFormConf);
+  }
+
+  if (preferredFoci.length === 0)
+    throw new Error(
+      `No livekit_service_url is configured so we could not create a focus.
+    Currently we skip computing a focus based on other users in the room.`,
+    );
+
+  return preferredFoci;
+
+  // TODO: we want to do something like this:
+  //
+  // const focusOtherMembers = await focusFromOtherMembers(
+  //   rtcSession,
+  //   livekitAlias,
+  // );
+  // if (focusOtherMembers) preferredFoci.push(focusOtherMembers);
+}
+
+export async function enterRTCSession(
   rtcSession: MatrixRTCSession,
   encryptMedia: boolean,
-): void {
+): Promise<void> {
   PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date());
   PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId);
 
@@ -47,8 +108,11 @@ export function enterRTCSession(
 
   // right now we assume everything is a room-scoped call
   const livekitAlias = rtcSession.room.roomId;
-
-  rtcSession.joinRoomSession([makeFocus(livekitAlias)], encryptMedia);
+  rtcSession.joinRoomSession(
+    await makePreferredLivekitFoci(rtcSession, livekitAlias),
+    makeActiveFocus(),
+    { manageMediaKeys: encryptMedia },
+  );
 }
 
 const widgetPostHangupProcedure = async (

src/useMatrixRTCSessionJoinState.ts

@@ -26,14 +26,17 @@ export function useMatrixRTCSessionJoinState(
 ): boolean {
   const [isJoined, setJoined] = useState(rtcSession.isJoined());
 
-  const onJoinStateChanged = useCallback(() => {
-    logger.info(
-      `Session in room ${rtcSession.room.roomId} changed to ${
-        rtcSession.isJoined() ? "joined" : "left"
-      }`,
-    );
-    setJoined(rtcSession.isJoined());
-  }, [rtcSession]);
+  const onJoinStateChanged = useCallback(
+    (isJoined: boolean) => {
+      logger.info(
+        `Session in room ${rtcSession.room.roomId} changed to ${
+          isJoined ? "joined" : "left"
+        }`,
+      );
+      setJoined(isJoined);
+    },
+    [rtcSession],
+  );
 
   useEffect(() => {
     rtcSession.on(MatrixRTCSessionEvent.JoinStateChanged, onJoinStateChanged);