From 6adf16dd3f7494f8616e5049c6c3d064b8ddbdf4 Mon Sep 17 00:00:00 2001 From: charliesantos Date: Wed, 27 Sep 2023 14:30:54 -0700 Subject: [PATCH 1/5] AudioProcessor interface --- lib/twilio.ts | 3 ++- lib/twilio/audioprocessor.ts | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 lib/twilio/audioprocessor.ts diff --git a/lib/twilio.ts b/lib/twilio.ts index 45189503..72dbab6e 100644 --- a/lib/twilio.ts +++ b/lib/twilio.ts @@ -2,10 +2,11 @@ * @packageDocumentation * @internalapi */ +import AudioProcessor from './twilio/audioprocessor'; import Call from './twilio/call'; import Device from './twilio/device'; import * as TwilioError from './twilio/errors'; import { Logger } from './twilio/log'; import { PreflightTest } from './twilio/preflight/preflight'; -export { Call, Device, PreflightTest, Logger, TwilioError }; +export { AudioProcessor, Call, Device, PreflightTest, Logger, TwilioError }; diff --git a/lib/twilio/audioprocessor.ts b/lib/twilio/audioprocessor.ts new file mode 100644 index 00000000..d0bf9e96 --- /dev/null +++ b/lib/twilio/audioprocessor.ts @@ -0,0 +1,34 @@ +/** + * @packageDocumentation + * @module Voice + */ + +/** + * Represents an AudioProcessor object that receives an audio stream for processing. + * @publicapi + */ +interface AudioProcessor { + /** + * Called whenever the active input audio stream is updated + * and is ready for processing such as adding audio filters + * or removing background noise. + * Use this method to initiate your audio processing pipeline. + * + * @param stream The input audio stream. + * @returns The modified input audio stream after applying filters. + */ + createProcessedStream(stream: MediaStream): Promise; + + /** + * Called after the processed stream has been destroyed. + * This happens whenever the current input stream is updated. + * Use this method to run any necessary teardown routines + * needed by your audio processing pipeline. + * + * @param stream The torn down processed audio stream. + * @returns + */ + destroyProcessedStream(stream: MediaStream): Promise; +} + +export default AudioProcessor; From c8e1657ba558150996dd9e0a9d4e3b549f14dbda Mon Sep 17 00:00:00 2001 From: charliesantos Date: Wed, 27 Sep 2023 15:31:39 -0700 Subject: [PATCH 2/5] Adding add and remove api --- lib/twilio/audiohelper.ts | 47 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/lib/twilio/audiohelper.ts b/lib/twilio/audiohelper.ts index e3011b5b..41ae6fa0 100644 --- a/lib/twilio/audiohelper.ts +++ b/lib/twilio/audiohelper.ts @@ -3,6 +3,7 @@ * @module Voice */ import { EventEmitter } from 'events'; +import AudioProcessor from './audioprocessor'; import Device from './device'; import { InvalidArgumentError, NotSupportedError } from './errors'; import Log from './log'; @@ -147,6 +148,11 @@ class AudioHelper extends EventEmitter { */ private _onActiveInputChanged: (stream: MediaStream | null) => Promise; + /** + * Internal reference to the added AudioProcessor + */ + private _processor: AudioProcessor | null; + /** * A record of unknown devices (Devices without labels) */ @@ -342,6 +348,35 @@ class AudioHelper extends EventEmitter { }); } + /** + * Adds an {@link AudioProcessor}. If an {@link AudioProcessor} exists in the + * {@link AudioHelper}, the {@link AudioHelper} routes the input audio stream + * to the {@link AudioProcessor}. + * + * The {@link AudioHelper} guarantees that this audio stream is always + * the active input audio stream and will automatically + * update whenever the user's preference changes. + * + * Only one {@link AudioProcessor} can be added at this time. + * @param processor + */ + addProcessor(processor: AudioProcessor): void { + if (this._processor) { + throw new NotSupportedError('Adding multiple AudioProcessors is not supported at this time.'); + } + + if (!processor) { + throw new InvalidArgumentError('Missing AudioProcessor argument.'); + } + + if (typeof processor.createProcessedStream !== 'function' || + typeof processor.destroyProcessedStream !== 'function') { + throw new InvalidArgumentError('Missing createProcessedStream or destroyProcessedStream.'); + } + + this._processor = processor; + } + /** * Enable or disable the disconnect sound. * @param doEnable Passing `true` will enable the sound and `false` will disable the sound. @@ -372,6 +407,18 @@ class AudioHelper extends EventEmitter { return this._maybeEnableSound(Device.SoundName.Outgoing, doEnable); } + /** + * Removes an AudioProcessor. + * @param processor + */ + removeProcessor(processor: AudioProcessor): void { + if (this._processor !== processor) { + throw new InvalidArgumentError('Cannot remove an AudioProcessor that has not been previously added.'); + } + + this._processor = null; + }; + /** * Set the MediaTrackConstraints to be applied on every getUserMedia call for new input * device audio. Any deviceId specified here will be ignored. Instead, device IDs should From 959f583b41f1af559e5b2b55a41e79c05e99f51d Mon Sep 17 00:00:00 2001 From: charliesantos Date: Wed, 27 Sep 2023 15:37:39 -0700 Subject: [PATCH 3/5] lint --- lib/twilio/audiohelper.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/twilio/audiohelper.ts b/lib/twilio/audiohelper.ts index 41ae6fa0..6403f493 100644 --- a/lib/twilio/audiohelper.ts +++ b/lib/twilio/audiohelper.ts @@ -417,7 +417,7 @@ class AudioHelper extends EventEmitter { } this._processor = null; - }; + } /** * Set the MediaTrackConstraints to be applied on every getUserMedia call for new input From f48ff3590a415e84509313e2ac2327f69f1d107c Mon Sep 17 00:00:00 2001 From: charliesantos Date: Mon, 2 Oct 2023 12:59:11 -0700 Subject: [PATCH 4/5] Address PR reviews --- lib/twilio/audiohelper.ts | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/lib/twilio/audiohelper.ts b/lib/twilio/audiohelper.ts index 6403f493..3aa09eea 100644 --- a/lib/twilio/audiohelper.ts +++ b/lib/twilio/audiohelper.ts @@ -349,13 +349,8 @@ class AudioHelper extends EventEmitter { } /** - * Adds an {@link AudioProcessor}. If an {@link AudioProcessor} exists in the - * {@link AudioHelper}, the {@link AudioHelper} routes the input audio stream - * to the {@link AudioProcessor}. - * - * The {@link AudioHelper} guarantees that this audio stream is always - * the active input audio stream and will automatically - * update whenever the user's preference changes. + * Adds an {@link AudioProcessor} object which will receive + * the input audio stream before sending to Twilio. * * Only one {@link AudioProcessor} can be added at this time. * @param processor @@ -365,13 +360,16 @@ class AudioHelper extends EventEmitter { throw new NotSupportedError('Adding multiple AudioProcessors is not supported at this time.'); } - if (!processor) { + if (typeof processor !== 'object' || processor === null) { throw new InvalidArgumentError('Missing AudioProcessor argument.'); } - if (typeof processor.createProcessedStream !== 'function' || - typeof processor.destroyProcessedStream !== 'function') { - throw new InvalidArgumentError('Missing createProcessedStream or destroyProcessedStream.'); + if (typeof processor.createProcessedStream !== 'function') { + throw new InvalidArgumentError('Missing createProcessedStream() method.'); + } + + if (typeof processor.destroyProcessedStream !== 'function') { + throw new InvalidArgumentError('Missing destroyProcessedStream() method.'); } this._processor = processor; From f7113327d560e0a4ec31c3f7949d00dad8de2443 Mon Sep 17 00:00:00 2001 From: Charlemagne Santos Date: Mon, 2 Oct 2023 13:26:00 -0700 Subject: [PATCH 5/5] Update audiohelper.ts --- lib/twilio/audiohelper.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/twilio/audiohelper.ts b/lib/twilio/audiohelper.ts index 3aa09eea..58c7fe49 100644 --- a/lib/twilio/audiohelper.ts +++ b/lib/twilio/audiohelper.ts @@ -349,8 +349,9 @@ class AudioHelper extends EventEmitter { } /** - * Adds an {@link AudioProcessor} object which will receive - * the input audio stream before sending to Twilio. + * Adds an AudioProcessor object. + * The AudioHelper will route the input audio stream through the processor + * before sending the audio stream to Twilio. * * Only one {@link AudioProcessor} can be added at this time. * @param processor