From a811aed26e2c1dfbac2436e2fe646c3d47dcc2af Mon Sep 17 00:00:00 2001 From: charliesantos Date: Mon, 6 Nov 2023 15:53:32 -0800 Subject: [PATCH] VBLOCKS-2032 | AudioProcessor insights and reusing AudioHelper --- lib/twilio/audiohelper.ts | 68 +++++++++++++++++++---- lib/twilio/audioprocessoreventobserver.ts | 36 ++++++++++++ lib/twilio/device.ts | 30 ++++++---- package-lock.json | 4 +- 4 files changed, 114 insertions(+), 24 deletions(-) create mode 100644 lib/twilio/audioprocessoreventobserver.ts diff --git a/lib/twilio/audiohelper.ts b/lib/twilio/audiohelper.ts index 6bb5c5eb..f4fda97c 100644 --- a/lib/twilio/audiohelper.ts +++ b/lib/twilio/audiohelper.ts @@ -4,6 +4,7 @@ */ import { EventEmitter } from 'events'; import AudioProcessor from './audioprocessor'; +import { AudioProcessorEventObserver } from './audioprocessoreventobserver'; import Device from './device'; import { InvalidArgumentError, NotSupportedError } from './errors'; import Log from './log'; @@ -90,6 +91,11 @@ class AudioHelper extends EventEmitter { */ private _audioContext?: AudioContext; + /** + * The AudioProcessorEventObserver instance to use + */ + private _audioProcessorEventObserver: AudioProcessorEventObserver; + /** * The audio stream of the default device. * This is populated when _openDefaultDeviceWithConstraints is called, @@ -184,12 +190,10 @@ class AudioHelper extends EventEmitter { * @private * @param onActiveOutputsChanged - A callback to be called when the user changes the active output devices. * @param onActiveInputChanged - A callback to be called when the user changes the active input device. - * @param getUserMedia - The getUserMedia method to use. * @param [options] */ constructor(onActiveOutputsChanged: (type: 'ringtone' | 'speaker', outputIds: string[]) => Promise, onActiveInputChanged: (stream: MediaStream | null) => Promise, - getUserMedia: (constraints: MediaStreamConstraints) => Promise, options?: AudioHelper.Options) { super(); @@ -198,7 +202,9 @@ class AudioHelper extends EventEmitter { setSinkId: typeof HTMLAudioElement !== 'undefined' && (HTMLAudioElement.prototype as any).setSinkId, }, options); - this._getUserMedia = getUserMedia; + this._updateUserOptions(options); + + this._audioProcessorEventObserver = options.audioProcessorEventObserver; this._mediaDevices = options.mediaDevices || navigator.mediaDevices; this._onActiveInputChanged = onActiveInputChanged; this._enumerateDevices = typeof options.enumerateDevices === 'function' @@ -262,11 +268,16 @@ class AudioHelper extends EventEmitter { } /** - * Current state of the enabled sounds + * Destroy this AudioHelper instance * @private */ - _getEnabledSounds(): Record { - return this._enabledSounds; + _destroy(): void { + this._stopDefaultInputDeviceStream(); + this._stopSelectedInputDeviceStream(); + this._destroyProcessedStream(); + this._maybeStopPollingVolume(); + this.removeAllListeners(); + this._unbind(); } /** @@ -399,6 +410,19 @@ class AudioHelper extends EventEmitter { }); } + /** + * Update AudioHelper options that can be changed by the user + * @private + */ + _updateUserOptions(options: AudioHelper.Options): void { + if (typeof options.enumerateDevices === 'function') { + this._enumerateDevices = options.enumerateDevices; + } + if (typeof options.getUserMedia === 'function') { + this._getUserMedia = options.getUserMedia; + } + } + /** * Adds an {@link AudioProcessor} object. * The AudioHelper will route the input audio stream through the processor @@ -427,6 +451,7 @@ class AudioHelper extends EventEmitter { this._log.debug('Adding processor'); this._processor = processor; this._restartStreams(); + this._audioProcessorEventObserver.emit('add'); } /** @@ -478,6 +503,7 @@ class AudioHelper extends EventEmitter { this._destroyProcessedStream(); this._processor = null; this._restartStreams(); + this._audioProcessorEventObserver.emit('remove'); } /** @@ -543,6 +569,7 @@ class AudioHelper extends EventEmitter { this._processedStream.getTracks().forEach(track => track.stop()); this._processedStream = null; this._processor.destroyProcessedStream(processedStream); + this._audioProcessorEventObserver.emit('destroy'); } } @@ -596,6 +623,7 @@ class AudioHelper extends EventEmitter { this._log.debug('Creating processed stream'); return this._processor.createProcessedStream(stream).then((processedStream: MediaStream) => { this._processedStream = processedStream; + this._audioProcessorEventObserver.emit('create'); return this._processedStream; }); } @@ -659,9 +687,7 @@ class AudioHelper extends EventEmitter { this._log.debug('Replacing with new stream.'); if (this._selectedInputDeviceStream) { this._log.debug('Old stream detected. Stopping tracks.'); - this._selectedInputDeviceStream.getTracks().forEach(track => { - track.stop(); - }); + this._stopSelectedInputDeviceStream(); } this._selectedInputDeviceStream = stream; @@ -714,9 +740,7 @@ class AudioHelper extends EventEmitter { // If the currently active track is still in readyState `live`, gUM may return the same track // rather than returning a fresh track. this._log.debug('Same track detected on setInputDevice, stopping old tracks.'); - this._selectedInputDeviceStream.getTracks().forEach(track => { - track.stop(); - }); + this._stopSelectedInputDeviceStream(); } // Release the default device in case it was created previously @@ -739,6 +763,16 @@ class AudioHelper extends EventEmitter { }); } + /** + * Stop the selected audio stream + */ + private _stopSelectedInputDeviceStream(): void { + if (this._selectedInputDeviceStream) { + this._log.debug('Stopping selected device stream'); + this._selectedInputDeviceStream.getTracks().forEach(track => track.stop()); + } + } + /** * Update a set of devices. * @param updatedDevices - An updated list of available Devices @@ -886,6 +920,11 @@ namespace AudioHelper { */ audioContext?: AudioContext; + /** + * AudioProcessorEventObserver to use + */ + audioProcessorEventObserver: AudioProcessorEventObserver, + /** * Whether each sound is enabled. */ @@ -896,6 +935,11 @@ namespace AudioHelper { */ enumerateDevices?: any; + /** + * The getUserMedia method to use + */ + getUserMedia: (constraints: MediaStreamConstraints) => Promise, + /** * A custom MediaDevices instance to use. */ diff --git a/lib/twilio/audioprocessoreventobserver.ts b/lib/twilio/audioprocessoreventobserver.ts new file mode 100644 index 00000000..42204287 --- /dev/null +++ b/lib/twilio/audioprocessoreventobserver.ts @@ -0,0 +1,36 @@ +/** + * @packageDocumentation + * @module Voice + * @internalapi + */ + +import { EventEmitter } from 'events'; +import Log from './log'; + +/** + * AudioProcessorEventObserver observes {@link AudioProcessor} + * related operations and re-emits them as generic events. + * @private + */ +export class AudioProcessorEventObserver extends EventEmitter { + + private _log: Log = Log.getInstance(); + + constructor() { + super(); + this._log.debug('Creating AudioProcessorEventObserver instance'); + this.on('add', () => this._reEmitEvent('add')); + this.on('remove', () => this._reEmitEvent('remove')); + this.on('create', () => this._reEmitEvent('create-processed-stream')); + this.on('destroy', () => this._reEmitEvent('destroy-processed-stream')); + } + + destroy(): void { + this.removeAllListeners(); + } + + private _reEmitEvent(name: string): void { + this._log.debug(`AudioProcessor:${name}`); + this.emit('event', { name, group: 'audio-processor' }); + } +} diff --git a/lib/twilio/device.ts b/lib/twilio/device.ts index c3a0bdc8..8a99f226 100644 --- a/lib/twilio/device.ts +++ b/lib/twilio/device.ts @@ -7,6 +7,7 @@ import { EventEmitter } from 'events'; import { levels as LogLevels, LogLevelDesc } from 'loglevel'; import AudioHelper from './audiohelper'; +import { AudioProcessorEventObserver } from './audioprocessoreventobserver'; import Call from './call'; import * as C from './constants'; import DialtonePlayer from './dialtonePlayer'; @@ -298,6 +299,11 @@ class Device extends EventEmitter { */ private _audio: AudioHelper | null = null; + /** + * The AudioProcessorEventObserver instance to use + */ + private _audioProcessorEventObserver: AudioProcessorEventObserver | null = null; + /** * {@link Device._confirmClose} bound to the specific {@link Device} instance. */ @@ -589,13 +595,10 @@ class Device extends EventEmitter { this.disconnectAll(); this._stopRegistrationTimer(); - if (this._audio) { - this._audio._unbind(); - } - this._destroyStream(); this._destroyPublisher(); this._destroyAudioHelper(); + this._audioProcessorEventObserver?.destroy(); if (this._networkInformation && typeof this._networkInformation.removeEventListener === 'function') { this._networkInformation.removeEventListener('change', this._publishNetworkChange); @@ -887,8 +890,7 @@ class Device extends EventEmitter { */ private _destroyAudioHelper() { if (!this._audio) { return; } - - this._audio.removeAllListeners(); + this._audio._destroy(); this._audio = null; } @@ -1325,21 +1327,29 @@ class Device extends EventEmitter { * Set up an audio helper for usage by this {@link Device}. */ private _setupAudioHelper(): void { + if (!this._audioProcessorEventObserver) { + this._audioProcessorEventObserver = new AudioProcessorEventObserver(); + this._audioProcessorEventObserver.on('event', ({ name, group }) => { + this._publisher.info(group, name, {}, this._activeCall); + }); + } + const audioOptions: AudioHelper.Options = { audioContext: Device.audioContext, + audioProcessorEventObserver: this._audioProcessorEventObserver, enumerateDevices: this._options.enumerateDevices, + getUserMedia: this._options.getUserMedia || getUserMedia, }; if (this._audio) { - this._log.info('Found existing audio helper; destroying...'); - audioOptions.enabledSounds = this._audio._getEnabledSounds(); - this._destroyAudioHelper(); + this._log.info('Found existing audio helper; updating options...'); + this._audio._updateUserOptions(audioOptions); + return; } this._audio = new (this._options.AudioHelper || AudioHelper)( this._updateSinkIds, this._updateInputStream, - this._options.getUserMedia || getUserMedia, audioOptions, ); diff --git a/package-lock.json b/package-lock.json index 3062d459..694d05e3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@twilio/voice-sdk", - "version": "2.8.0-dev", + "version": "2.8.1-dev", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@twilio/voice-sdk", - "version": "2.8.0-dev", + "version": "2.8.1-dev", "license": "Apache-2.0", "dependencies": { "@twilio/voice-errors": "1.4.0",