diff --git a/spec/unit/utils.spec.ts b/spec/unit/utils.spec.ts index 9611511b40d..12a83b235df 100644 --- a/spec/unit/utils.spec.ts +++ b/spec/unit/utils.spec.ts @@ -493,4 +493,68 @@ describe("utils", function() { expect(deepSortedObjectEntries(input)).toMatchObject(output); }); }); + + describe("recursivelyAssign", () => { + it("doesn't override with null/undefined", () => { + const result = utils.recursivelyAssign( + { + string: "Hello world", + object: {}, + float: 0.1, + }, { + string: null, + object: undefined, + }, + true, + ); + + expect(result).toStrictEqual({ + string: "Hello world", + object: {}, + float: 0.1, + }); + }); + + it("assigns recursively", () => { + const result = utils.recursivelyAssign( + { + number: 42, + object: { + message: "Hello world", + day: "Monday", + langs: { + compiled: ["c++"], + }, + }, + thing: "string", + }, { + number: 2, + object: { + message: "How are you", + day: "Friday", + langs: { + compiled: ["c++", "c"], + }, + }, + thing: { + aSubThing: "something", + }, + }, + ); + + expect(result).toStrictEqual({ + number: 2, + object: { + message: "How are you", + day: "Friday", + langs: { + compiled: ["c++", "c"], + }, + }, + thing: { + aSubThing: "something", + }, + }); + }); + }); }); diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index 69a3553b01c..400c5ecb360 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -321,16 +321,19 @@ describe('Call', function() { [SDPStreamMetadataKey]: { "stream_id": { purpose: SDPStreamMetadataPurpose.Usermedia, + audio_muted: true, + video_muted: false, }, }, }; }, }); - call.pushRemoteFeed({ id: "stream_id" }); - expect(call.getFeeds().find((feed) => { - return feed.stream.id === "stream_id"; - })?.purpose).toBe(SDPStreamMetadataPurpose.Usermedia); + call.pushRemoteFeed({ id: "stream_id", getAudioTracks: () => ["track1"], getVideoTracks: () => ["track1"] }); + const feed = call.getFeeds().find((feed) => feed.stream.id === "stream_id"); + expect(feed?.purpose).toBe(SDPStreamMetadataPurpose.Usermedia); + expect(feed?.isAudioMuted()).toBeTruthy(); + expect(feed?.isVideoMuted()).not.toBeTruthy(); }); it("should fallback to replaceTrack() if the other side doesn't support SPDStreamMetadata", async () => { diff --git a/src/@types/event.ts b/src/@types/event.ts index 42c7d0529c7..56ac83d8a38 100644 --- a/src/@types/event.ts +++ b/src/@types/event.ts @@ -53,6 +53,8 @@ export enum EventType { CallReject = "m.call.reject", CallSelectAnswer = "m.call.select_answer", CallNegotiate = "m.call.negotiate", + CallSDPStreamMetadataChanged = "m.call.sdp_stream_metadata_changed", + CallSDPStreamMetadataChangedPrefix = "org.matrix.call.sdp_stream_metadata_changed", CallReplaces = "m.call.replaces", CallAssertedIdentity = "m.call.asserted_identity", CallAssertedIdentityPrefix = "org.matrix.call.asserted_identity", diff --git a/src/utils.ts b/src/utils.ts index 5abed06af2a..b5701df67a7 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -693,3 +693,25 @@ const collator = new Intl.Collator(); export function compare(a: string, b: string): number { return collator.compare(a, b); } + +/** + * This function is similar to Object.assign() but it assigns recursively and + * allows you to ignore nullish values from the source + * + * @param {Object} target + * @param {Object} source + * @returns the target object + */ +export function recursivelyAssign(target: Object, source: Object, ignoreNullish = false): any { + for (const [sourceKey, sourceValue] of Object.entries(source)) { + if (target[sourceKey] instanceof Object && sourceValue) { + recursivelyAssign(target[sourceKey], sourceValue); + continue; + } + if ((sourceValue !== null && sourceValue !== undefined) || !ignoreNullish) { + target[sourceKey] = sourceValue; + continue; + } + } + return target; +} diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index dc8837ce37a..a18ccfae7d2 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -36,6 +36,7 @@ import { SDPStreamMetadataPurpose, SDPStreamMetadata, SDPStreamMetadataKey, + MCallSDPStreamMetadataChanged, } from './callEventTypes'; import { CallFeed } from './callFeed'; @@ -353,8 +354,6 @@ export class MatrixCall extends EventEmitter { this.makingOffer = false; this.remoteOnHold = false; - this.micMuted = false; - this.vidMuted = false; this.feeds = []; @@ -402,6 +401,14 @@ export class MatrixCall extends EventEmitter { return this.remoteAssertedIdentity; } + public get localUsermediaFeed(): CallFeed { + return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); + } + + private getFeedByStreamId(streamId: string): CallFeed { + return this.getFeeds().find((feed) => feed.stream.id === streamId); + } + /** * Returns an array of all CallFeeds * @returns {Array} CallFeeds @@ -431,10 +438,12 @@ export class MatrixCall extends EventEmitter { * @returns {SDPStreamMetadata} localSDPStreamMetadata */ private getLocalSDPStreamMetadata(): SDPStreamMetadata { - const metadata = {}; + const metadata: SDPStreamMetadata = {}; for (const localFeed of this.getLocalFeeds()) { metadata[localFeed.stream.id] = { purpose: localFeed.purpose, + audio_muted: localFeed.isAudioMuted(), + video_muted: localFeed.isVideoMuted(), }; } logger.debug("Got local SDPStreamMetadata", metadata); @@ -459,6 +468,8 @@ export class MatrixCall extends EventEmitter { const userId = this.getOpponentMember().userId; const purpose = this.remoteSDPStreamMetadata[stream.id].purpose; + const audioMuted = this.remoteSDPStreamMetadata[stream.id].audio_muted; + const videoMuted = this.remoteSDPStreamMetadata[stream.id].video_muted; if (!purpose) { logger.warn(`Ignoring stream with id ${stream.id} because we didn't get any metadata about it`); @@ -471,7 +482,7 @@ export class MatrixCall extends EventEmitter { if (existingFeed) { existingFeed.setNewStream(stream); } else { - this.feeds.push(new CallFeed(stream, userId, purpose, this.client, this.roomId)); + this.feeds.push(new CallFeed(stream, userId, purpose, this.client, this.roomId, audioMuted, videoMuted)); this.emit(CallEvent.FeedsChanged, this.feeds); } @@ -498,11 +509,11 @@ export class MatrixCall extends EventEmitter { // Try to find a feed with the same stream id as the new stream, // if we find it replace the old stream with the new one - const feed = this.feeds.find((feed) => feed.stream.id === stream.id); + const feed = this.getFeedByStreamId(stream.id); if (feed) { feed.setNewStream(stream); } else { - this.feeds.push(new CallFeed(stream, userId, purpose, this.client, this.roomId)); + this.feeds.push(new CallFeed(stream, userId, purpose, this.client, this.roomId, false, false)); this.emit(CallEvent.FeedsChanged, this.feeds); } @@ -517,7 +528,7 @@ export class MatrixCall extends EventEmitter { if (existingFeed) { existingFeed.setNewStream(stream); } else { - this.feeds.push(new CallFeed(stream, userId, purpose, this.client, this.roomId)); + this.feeds.push(new CallFeed(stream, userId, purpose, this.client, this.roomId, false, false)); this.emit(CallEvent.FeedsChanged, this.feeds); } @@ -555,7 +566,7 @@ export class MatrixCall extends EventEmitter { private deleteFeedByStream(stream: MediaStream) { logger.debug(`Removing feed with stream id ${stream.id}`); - const feed = this.feeds.find((feed) => feed.stream.id === stream.id); + const feed = this.getFeedByStreamId(stream.id); if (!feed) { logger.warn(`Didn't find the feed with stream id ${stream.id} to delete`); return; @@ -605,7 +616,7 @@ export class MatrixCall extends EventEmitter { const sdpStreamMetadata = invite[SDPStreamMetadataKey]; if (sdpStreamMetadata) { - this.remoteSDPStreamMetadata = sdpStreamMetadata; + this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); } else { logger.debug("Did not get any SDPStreamMetadata! Can not send/receive multiple streams"); } @@ -891,7 +902,7 @@ export class MatrixCall extends EventEmitter { * @param {boolean} muted True to mute the outbound video. */ setLocalVideoMuted(muted: boolean) { - this.vidMuted = muted; + this.localUsermediaFeed?.setVideoMuted(muted); this.updateMuteStatus(); } @@ -905,8 +916,7 @@ export class MatrixCall extends EventEmitter { * (including if the call is not set up yet). */ isLocalVideoMuted(): boolean { - if (this.type === CallType.Voice) return true; - return this.vidMuted; + return this.localUsermediaFeed?.isVideoMuted(); } /** @@ -914,7 +924,7 @@ export class MatrixCall extends EventEmitter { * @param {boolean} muted True to mute the mic. */ setMicrophoneMuted(muted: boolean) { - this.micMuted = muted; + this.localUsermediaFeed?.setAudioMuted(muted); this.updateMuteStatus(); } @@ -928,7 +938,7 @@ export class MatrixCall extends EventEmitter { * is not set up yet). */ isMicrophoneMuted(): boolean { - return this.micMuted; + return this.localUsermediaFeed?.isAudioMuted(); } /** @@ -991,14 +1001,14 @@ export class MatrixCall extends EventEmitter { } private updateMuteStatus() { - if (!this.localAVStream) { - return; - } + this.sendVoipEvent(EventType.CallSDPStreamMetadataChangedPrefix, { + [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(), + }); - const micShouldBeMuted = this.micMuted || this.remoteOnHold; - setTracksEnabled(this.localAVStream.getAudioTracks(), !micShouldBeMuted); + const micShouldBeMuted = this.localUsermediaFeed?.isAudioMuted() || this.remoteOnHold; + const vidShouldBeMuted = this.localUsermediaFeed?.isVideoMuted() || this.remoteOnHold; - const vidShouldBeMuted = this.vidMuted || this.remoteOnHold; + setTracksEnabled(this.localAVStream.getAudioTracks(), !micShouldBeMuted); setTracksEnabled(this.localAVStream.getVideoTracks(), !vidShouldBeMuted); } @@ -1214,7 +1224,7 @@ export class MatrixCall extends EventEmitter { const sdpStreamMetadata = event.getContent()[SDPStreamMetadataKey]; if (sdpStreamMetadata) { - this.remoteSDPStreamMetadata = sdpStreamMetadata; + this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); } else { logger.warn("Did not get any SDPStreamMetadata! Can not send/receive multiple streams"); } @@ -1289,9 +1299,9 @@ export class MatrixCall extends EventEmitter { const prevLocalOnHold = this.isLocalOnHold(); - const metadata = event.getContent()[SDPStreamMetadataKey]; - if (metadata) { - this.remoteSDPStreamMetadata = metadata; + const sdpStreamMetadata = event.getContent()[SDPStreamMetadataKey]; + if (sdpStreamMetadata) { + this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); } else { logger.warn("Received negotiation event without SDPStreamMetadata!"); } @@ -1321,6 +1331,22 @@ export class MatrixCall extends EventEmitter { } } + private updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata): void { + this.remoteSDPStreamMetadata = utils.recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true); + for (const feed of this.getRemoteFeeds()) { + const streamId = feed.stream.id; + feed.setAudioMuted(this.remoteSDPStreamMetadata[streamId]?.audio_muted); + feed.setVideoMuted(this.remoteSDPStreamMetadata[streamId]?.video_muted); + feed.purpose = this.remoteSDPStreamMetadata[streamId]?.purpose; + } + } + + public onSDPStreamMetadataChangedReceived(event: MatrixEvent): void { + const content = event.getContent(); + const metadata = content[SDPStreamMetadataKey]; + this.updateRemoteSDPStreamMetadata(metadata); + } + async onAssertedIdentityReceived(event: MatrixEvent) { if (!event.getContent().asserted_identity) return; diff --git a/src/webrtc/callEventHandler.ts b/src/webrtc/callEventHandler.ts index 263ddbf9bcf..96b233e0f1d 100644 --- a/src/webrtc/callEventHandler.ts +++ b/src/webrtc/callEventHandler.ts @@ -297,6 +297,18 @@ export class CallEventHandler { } call.onAssertedIdentityReceived(event); + } else if ( + event.getType() === EventType.CallSDPStreamMetadataChanged || + event.getType() === EventType.CallSDPStreamMetadataChangedPrefix + ) { + if (!call) return; + + if (event.getContent().party_id === call.ourPartyId) { + // Ignore remote echo + return; + } + + call.onSDPStreamMetadataChangedReceived(event); } } } diff --git a/src/webrtc/callEventTypes.ts b/src/webrtc/callEventTypes.ts index c243b8055f7..7adf640e8d4 100644 --- a/src/webrtc/callEventTypes.ts +++ b/src/webrtc/callEventTypes.ts @@ -11,6 +11,8 @@ export enum SDPStreamMetadataPurpose { export interface SDPStreamMetadataObject { purpose: SDPStreamMetadataPurpose; + audio_muted: boolean; + video_muted: boolean; } export interface SDPStreamMetadata { @@ -41,6 +43,10 @@ export interface MCallOfferNegotiate { [SDPStreamMetadataKey]: SDPStreamMetadata; } +export interface MCallSDPStreamMetadataChanged { + [SDPStreamMetadataKey]: SDPStreamMetadata; +} + export interface MCallReplacesTarget { id: string; display_name: string; diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index 69d6170abeb..b82c68535bd 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -21,6 +21,7 @@ import { RoomMember } from "../models/room-member"; export enum CallFeedEvent { NewStream = "new_stream", + MuteStateChanged = "mute_state_changed" } export class CallFeed extends EventEmitter { @@ -30,6 +31,8 @@ export class CallFeed extends EventEmitter { public purpose: SDPStreamMetadataPurpose, private client: MatrixClient, private roomId: string, + private audioMuted: boolean, + private videoMuted: boolean, ) { super(); } @@ -51,15 +54,13 @@ export class CallFeed extends EventEmitter { return this.userId === this.client.getUserId(); } - // TODO: The two following methods should be later replaced - // by something that will also check if the remote is muted /** * Returns true if audio is muted or if there are no audio * tracks, otherwise returns false * @returns {boolean} is audio muted? */ public isAudioMuted(): boolean { - return this.stream.getAudioTracks().length === 0; + return this.stream.getAudioTracks().length === 0 || this.audioMuted; } /** @@ -69,7 +70,7 @@ export class CallFeed extends EventEmitter { */ public isVideoMuted(): boolean { // We assume only one video track - return this.stream.getVideoTracks().length === 0; + return this.stream.getVideoTracks().length === 0 || this.videoMuted; } /** @@ -81,4 +82,14 @@ export class CallFeed extends EventEmitter { this.stream = newStream; this.emit(CallFeedEvent.NewStream, this.stream); } + + public setAudioMuted(muted: boolean): void { + this.audioMuted = muted; + this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); + } + + public setVideoMuted(muted: boolean): void { + this.videoMuted = muted; + this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); + } }