From 90e74cce534098cfcd2a6b384fc238c5f554566f Mon Sep 17 00:00:00 2001 From: Florent Date: Thu, 3 Oct 2024 11:35:02 +0200 Subject: [PATCH] feature: add the possibility to rely on managedMediaSource on iOS devices add unit test for managed media source handle start and end stream event from managedMediaSource fix unit test react to event endStreaming and startStreaming to immediately trigger a new observation and to download the segments sooner review feedback restart Loading queue if it canLoad is updated add clearSignal to prevent memory leak remove undefined from canLoad only set disableRemotePlayback if it's defined by the browser fmt --- .../browser_compatibility_types.test.ts | 37 ++++++++++++++ src/compat/browser_compatibility_types.ts | 25 ++++++---- src/core/fetchers/segment/segment_queue.ts | 32 ++++++++++++ src/core/stream/adaptation/types.ts | 10 ++++ src/core/stream/period/types.ts | 10 ++++ .../representation/representation_stream.ts | 17 ++++++- src/core/stream/representation/types.ts | 10 ++++ .../prepare_source_buffer.ts | 25 +++++++++- .../init/multi_thread_content_initializer.ts | 50 ++++++++++++++++++- .../utils/create_core_playback_observer.ts | 21 ++++++++ .../init/utils/create_media_source.ts | 26 +++++++++- src/mse/main_media_source_interface.ts | 23 +++++++++ src/mse/types.ts | 28 +++++++++++ src/multithread_types.ts | 10 ++++ .../worker_playback_observer.ts | 10 ++++ 15 files changed, 318 insertions(+), 16 deletions(-) diff --git a/src/compat/__tests__/browser_compatibility_types.test.ts b/src/compat/__tests__/browser_compatibility_types.test.ts index acda8284a6..fd38f6ecfd 100644 --- a/src/compat/__tests__/browser_compatibility_types.test.ts +++ b/src/compat/__tests__/browser_compatibility_types.test.ts @@ -7,6 +7,7 @@ describe("compat - browser compatibility types", () => { MozMediaSource?: unknown; WebKitMediaSource?: unknown; MSMediaSource?: unknown; + ManagedMediaSource?: unknown; } const gs = globalScope as IFakeWindow; beforeEach(() => { @@ -22,11 +23,13 @@ describe("compat - browser compatibility types", () => { const origMozMediaSource = gs.MozMediaSource; const origWebKitMediaSource = gs.WebKitMediaSource; const origMSMediaSource = gs.MSMediaSource; + const origManagedMediaSource = gs.ManagedMediaSource; gs.MediaSource = { a: 1 }; gs.MozMediaSource = { a: 2 }; gs.WebKitMediaSource = { a: 3 }; gs.MSMediaSource = { a: 4 }; + gs.ManagedMediaSource = { a: 5 }; const { MediaSource_ } = await vi.importActual("../browser_compatibility_types"); expect(MediaSource_).toEqual({ a: 1 }); @@ -35,6 +38,7 @@ describe("compat - browser compatibility types", () => { gs.MozMediaSource = origMozMediaSource; gs.WebKitMediaSource = origWebKitMediaSource; gs.MSMediaSource = origMSMediaSource; + gs.ManagedMediaSource = origManagedMediaSource; }); it("should use MozMediaSource if defined and MediaSource is not", async () => { @@ -46,6 +50,7 @@ describe("compat - browser compatibility types", () => { const origMozMediaSource = gs.MozMediaSource; const origWebKitMediaSource = gs.WebKitMediaSource; const origMSMediaSource = gs.MSMediaSource; + const origManagedMediaSource = gs.ManagedMediaSource; gs.MediaSource = undefined; gs.MozMediaSource = { a: 2 }; @@ -59,6 +64,7 @@ describe("compat - browser compatibility types", () => { gs.MozMediaSource = origMozMediaSource; gs.WebKitMediaSource = origWebKitMediaSource; gs.MSMediaSource = origMSMediaSource; + gs.ManagedMediaSource = origManagedMediaSource; }); it("should use WebKitMediaSource if defined and MediaSource is not", async () => { @@ -70,6 +76,7 @@ describe("compat - browser compatibility types", () => { const origMozMediaSource = gs.MozMediaSource; const origWebKitMediaSource = gs.WebKitMediaSource; const origMSMediaSource = gs.MSMediaSource; + const origManagedMediaSource = gs.ManagedMediaSource; gs.MediaSource = undefined; gs.MozMediaSource = undefined; @@ -83,6 +90,7 @@ describe("compat - browser compatibility types", () => { gs.MozMediaSource = origMozMediaSource; gs.WebKitMediaSource = origWebKitMediaSource; gs.MSMediaSource = origMSMediaSource; + gs.ManagedMediaSource = origManagedMediaSource; }); it("should use MSMediaSource if defined and MediaSource is not", async () => { @@ -94,6 +102,7 @@ describe("compat - browser compatibility types", () => { const origMozMediaSource = gs.MozMediaSource; const origWebKitMediaSource = gs.WebKitMediaSource; const origMSMediaSource = gs.MSMediaSource; + const origManagedMediaSource = gs.ManagedMediaSource; gs.MediaSource = undefined; gs.MozMediaSource = undefined; @@ -107,5 +116,33 @@ describe("compat - browser compatibility types", () => { gs.MozMediaSource = origMozMediaSource; gs.WebKitMediaSource = origWebKitMediaSource; gs.MSMediaSource = origMSMediaSource; + gs.ManagedMediaSource = origManagedMediaSource; + }); + + it("should use ManagedMediaSource if defined and MediaSource is not", async () => { + vi.doMock("../../utils/is_node", () => ({ + default: false, + })); + + const origMediaSource = gs.MediaSource; + const origMozMediaSource = gs.MozMediaSource; + const origWebKitMediaSource = gs.WebKitMediaSource; + const origMSMediaSource = gs.MSMediaSource; + const origManagedMediaSource = gs.ManagedMediaSource; + + gs.MediaSource = undefined; + gs.MozMediaSource = undefined; + gs.WebKitMediaSource = undefined; + gs.MSMediaSource = undefined; + gs.ManagedMediaSource = { a: 5 }; + + const { MediaSource_ } = await vi.importActual("../browser_compatibility_types"); + expect(MediaSource_).toEqual({ a: 5 }); + + gs.MediaSource = origMediaSource; + gs.MozMediaSource = origMozMediaSource; + gs.WebKitMediaSource = origWebKitMediaSource; + gs.MSMediaSource = origMSMediaSource; + gs.ManagedMediaSource = origManagedMediaSource; }); }); diff --git a/src/compat/browser_compatibility_types.ts b/src/compat/browser_compatibility_types.ts index 3bfd5c90a0..adc957c639 100644 --- a/src/compat/browser_compatibility_types.ts +++ b/src/compat/browser_compatibility_types.ts @@ -16,7 +16,6 @@ import type { IListener } from "../utils/event_emitter"; import globalScope from "../utils/global_scope"; -import isNullOrUndefined from "../utils/is_null_or_undefined"; /** Regular MediaKeys type + optional functions present in IE11. */ interface ICompatMediaKeysConstructor { @@ -116,6 +115,8 @@ export interface IMediaSourceEventMap { sourceopen: Event; sourceended: Event; sourceclose: Event; + startstreaming: Event; + endstreaming: Event; } /** @@ -136,6 +137,7 @@ export interface IMediaSource extends IEventTarget { handle?: MediaProvider | IMediaSource | undefined; readyState: "closed" | "open" | "ended"; sourceBuffers: ISourceBufferList; + streaming?: boolean | undefined; addSourceBuffer(type: string): ISourceBuffer; clearLiveSeekableRange(): void; @@ -252,6 +254,7 @@ export interface IMediaElement extends IEventTarget { srcObject?: undefined | null | MediaProvider; textTracks: TextTrackList | never[]; volume: number; + disableRemotePlayback?: boolean; addTextTrack: (kind: TextTrackKind) => TextTrack; appendChild(x: T): void; @@ -407,15 +410,15 @@ const gs = globalScope as any; const MediaSource_: | { new (): IMediaSource; isTypeSupported(type: string): boolean } | undefined = - gs === undefined - ? undefined - : !isNullOrUndefined(gs.MediaSource) - ? gs.MediaSource - : !isNullOrUndefined(gs.MozMediaSource) - ? gs.MozMediaSource - : !isNullOrUndefined(gs.WebKitMediaSource) - ? gs.WebKitMediaSource - : gs.MSMediaSource; + gs?.MediaSource ?? + gs?.MozMediaSource ?? + gs?.WebKitMediaSource ?? + gs?.MSMediaSource ?? + gs?.ManagedMediaSource ?? + undefined; + +const isManagedMediaSource = + MediaSource_ !== undefined && MediaSource_ === gs?.ManagedMediaSource; /* eslint-enable */ /** List an HTMLMediaElement's possible values for its readyState property. */ @@ -448,4 +451,4 @@ export type { ICompatVTTCue, ICompatVTTCueConstructor, }; -export { MediaSource_, READY_STATES }; +export { MediaSource_, isManagedMediaSource, READY_STATES }; diff --git a/src/core/fetchers/segment/segment_queue.ts b/src/core/fetchers/segment/segment_queue.ts index 6341600b55..75cb79b62e 100644 --- a/src/core/fetchers/segment/segment_queue.ts +++ b/src/core/fetchers/segment/segment_queue.ts @@ -99,6 +99,8 @@ export default class SegmentQueue extends EventEmitter> * load segments for. * @param {boolean} hasInitSegment - Declare that an initialization segment * will need to be downloaded. + * @param {Object} canLoad - Indicates if the loading of new segment is authorized, + * it can be temporarily forbiddden if there is enough buffered data. * * A `SegmentQueue` ALWAYS wait for the initialization segment to be * loaded and parsed before parsing a media segment. @@ -116,6 +118,7 @@ export default class SegmentQueue extends EventEmitter> public resetForContent( content: ISegmentQueueContext, hasInitSegment: boolean, + canLoad: SharedReference, ): SharedReference { this._currentContentInfo?.currentCanceller.cancel(); const downloadQueue = new SharedReference({ @@ -136,9 +139,23 @@ export default class SegmentQueue extends EventEmitter> initSegmentRequest: null, mediaSegmentRequest: null, mediaSegmentAwaitingInitMetadata: null, + canLoad, }; this._currentContentInfo = currentContentInfo; + this._currentContentInfo.canLoad.onUpdate( + (val) => { + if (val) { + log.debug( + "SQ: Media segment can be loaded again, restarting queue.", + content.adaptation.type, + ); + this._restartMediaSegmentDownloadingQueue(currentContentInfo); + } + }, + { clearSignal: currentCanceller.signal }, + ); + // Listen for asked media segments downloadQueue.onUpdate( (queue) => { @@ -257,6 +274,10 @@ export default class SegmentQueue extends EventEmitter> const { downloadQueue, content, initSegmentInfoRef, currentCanceller } = contentInfo; const recursivelyRequestSegments = (): void => { + if (!contentInfo.canLoad.getValue()) { + log.debug("SQ: Segment fetching postponed because it cannot stream now."); + return; + } const { segmentQueue } = downloadQueue.getValue(); const startingSegment = segmentQueue[0]; if (currentCanceller !== null && currentCanceller.isUsed()) { @@ -681,4 +702,15 @@ interface ISegmentQueueContentInfo { * `null` if no segment is awaiting an init segment. */ mediaSegmentAwaitingInitMetadata: string | null; + /** + * Indicates whether the user agent believes it has enough buffered data to ensure + * uninterrupted playback for a meaningful period or needs more data. + * It also reflects whether the user agent can retrieve and buffer data in an + * energy-efficient manner while maintaining the desired memory usage. + * The value can be `undefined` if the user agent does not provide this indicator. + * `true` indicates that the buffer is low, and more data should be buffered. + * `false` indicates that there is enough buffered data, and no additional data needs + * to be buffered at this time. + */ + canLoad: SharedReference; } diff --git a/src/core/stream/adaptation/types.ts b/src/core/stream/adaptation/types.ts index 862466a416..3271bbeaa6 100644 --- a/src/core/stream/adaptation/types.ts +++ b/src/core/stream/adaptation/types.ts @@ -122,6 +122,16 @@ export interface IAdaptationStreamPlaybackObservation duration: number; /** Theoretical maximum position on the content that can currently be played. */ maximumPosition: number; + /** + * Indicates whether the user agent believes it has enough buffered data to ensure + * uninterrupted playback for a meaningful period or needs more data. + * It also reflects whether the user agent can retrieve and buffer data in an + * energy-efficient manner while maintaining the desired memory usage. + * `true` indicates that the buffer is low, and more data should be buffered. + * `false` indicates that there is enough buffered data, and no additional data needs + * to be buffered at this time. + */ + canStream: boolean; } /** Arguments given when creating a new `AdaptationStream`. */ diff --git a/src/core/stream/period/types.ts b/src/core/stream/period/types.ts index 67dbd7c756..bf3c98100c 100644 --- a/src/core/stream/period/types.ts +++ b/src/core/stream/period/types.ts @@ -96,6 +96,16 @@ export interface IPeriodStreamPlaybackObservation { * `null` if no buffer exists for that type of media. */ buffered: Record; + /** + * Indicates whether the user agent believes it has enough buffered data to ensure + * uninterrupted playback for a meaningful period or needs more data. + * It also reflects whether the user agent can retrieve and buffer data in an + * energy-efficient manner while maintaining the desired memory usage. + * `true` indicates that the buffer is low, and more data should be buffered. + * `false` indicates that there is enough buffered data, and no additional data needs + * to be buffered at this time. + */ + canStream: boolean; } /** Arguments required by the `PeriodStream`. */ diff --git a/src/core/stream/representation/representation_stream.ts b/src/core/stream/representation/representation_stream.ts index 4fd7aab17a..1a619eebfd 100644 --- a/src/core/stream/representation/representation_stream.ts +++ b/src/core/stream/representation/representation_stream.ts @@ -27,6 +27,7 @@ import config from "../../../config"; import log from "../../../log"; import type { ISegment } from "../../../manifest"; import objectAssign from "../../../utils/object_assign"; +import SharedReference from "../../../utils/reference"; import type { CancellationSignal } from "../../../utils/task_canceller"; import TaskCanceller, { CancellationError } from "../../../utils/task_canceller"; import type { @@ -209,8 +210,22 @@ export default function RepresentationStream( segmentsLoadingCanceller.signal, ); + const canStream = new SharedReference(true); + + playbackObserver.listen((observation) => { + const observationCanStream = observation.canStream ?? true; + if (canStream.getValue() !== observationCanStream) { + log.debug("Stream: observation.canStream updated to", observationCanStream); + canStream.setValue(observationCanStream); + } + }); + /** Emit the last scheduled downloading queue for segments. */ - const segmentsToLoadRef = segmentQueue.resetForContent(content, hasInitSegment); + const segmentsToLoadRef = segmentQueue.resetForContent( + content, + hasInitSegment, + canStream, + ); segmentsLoadingCanceller.signal.register(() => { segmentQueue.stop(); diff --git a/src/core/stream/representation/types.ts b/src/core/stream/representation/types.ts index 4c37c5cbfc..9380cb33dc 100644 --- a/src/core/stream/representation/types.ts +++ b/src/core/stream/representation/types.ts @@ -192,6 +192,16 @@ export interface IRepresentationStreamPlaybackObservation { paused: IPausedPlaybackObservation; /** Last "playback rate" asked by the user. */ speed: number; + /** + * Indicates whether the user agent believes it has enough buffered data to ensure + * uninterrupted playback for a meaningful period or needs more data. + * It also reflects whether the user agent can retrieve and buffer data in an + * energy-efficient manner while maintaining the desired memory usage. + * `true` indicates that the buffer is low, and more data should be buffered. + * `false` indicates that there is enough buffered data, and no additional data needs + * to be buffered at this time. + */ + canStream: boolean; } /** Pause-related information linked to an emitted Playback observation. */ diff --git a/src/experimental/tools/VideoThumbnailLoader/prepare_source_buffer.ts b/src/experimental/tools/VideoThumbnailLoader/prepare_source_buffer.ts index 5ad1782ff1..917ed56f97 100644 --- a/src/experimental/tools/VideoThumbnailLoader/prepare_source_buffer.ts +++ b/src/experimental/tools/VideoThumbnailLoader/prepare_source_buffer.ts @@ -14,7 +14,10 @@ * limitations under the License. */ -import { MediaSource_ } from "../../../compat/browser_compatibility_types"; +import { + isManagedMediaSource, + MediaSource_, +} from "../../../compat/browser_compatibility_types"; import type { IMediaElement } from "../../../compat/browser_compatibility_types"; import log from "../../../log"; import { resetMediaElement } from "../../../main_thread/init/utils/create_media_source"; @@ -66,6 +69,26 @@ export default function prepareSourceBuffer( resetMediaElement(videoElement, objectURL); }); } + if (isManagedMediaSource && "disableRemotePlayback" in videoElement) { + const disableRemotePlaybackPreviousValue = videoElement.disableRemotePlayback; + cleanUpSignal.register(() => { + /** + * Restore the disableRemotePlayback attribute to the previous value + * in order to leave the