From e651e8306661a2df5d39a6ecaa1f9e1a6ae4b12a Mon Sep 17 00:00:00 2001 From: Evan Burton Date: Thu, 16 Nov 2023 13:43:30 -0800 Subject: [PATCH 1/3] Add `videoPreference` config option for HDR/SDR VIDEO-RANGE selection and priority Resolves #2489 --- api-extractor/report/hls.js.api.md | 9 ++++ docs/API.md | 17 ++++++++ src/config.ts | 2 + src/controller/abr-controller.ts | 11 +++-- src/hls.ts | 2 + src/types/media-playlist.ts | 6 +++ src/utils/hdr.ts | 70 ++++++++++++++++++++++++++++++ src/utils/rendition-helper.ts | 37 ++++++++++++---- 8 files changed, 142 insertions(+), 12 deletions(-) create mode 100644 src/utils/hdr.ts diff --git a/api-extractor/report/hls.js.api.md b/api-extractor/report/hls.js.api.md index b54101eff27..075e15f22dd 100644 --- a/api-extractor/report/hls.js.api.md +++ b/api-extractor/report/hls.js.api.md @@ -3081,6 +3081,7 @@ export type RetryConfig = { // // @public (undocumented) export type SelectionPreferences = { + videoPreference?: VideoSelectionOption; audioPreference?: AudioSelectionOption; subtitlePreference?: SubtitleSelectionOption; }; @@ -3417,6 +3418,14 @@ export interface UserdataSample { // @public (undocumented) export type VariableMap = Record; +// Warning: (ae-missing-release-tag) "VideoSelectionOption" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type VideoSelectionOption = { + preferHDR?: boolean; + allowedVideoRanges?: Array; +}; + // (No @packageDocumentation comment for this package) ``` diff --git a/docs/API.md b/docs/API.md index ef8fc0d64fb..88f7ff4cf3e 100644 --- a/docs/API.md +++ b/docs/API.md @@ -79,6 +79,7 @@ See [API Reference](https://hlsjs-dev.video-dev.org/api-docs/) for a complete li - [`pLoader`](#ploader) - [`xhrSetup`](#xhrsetup) - [`fetchSetup`](#fetchsetup) + - [`videoPreference`](#videopreference) - [`audioPreference`](#audiopreference) - [`subtitlePreference`](#subtitlepreference) - [`abrController`](#abrcontroller) @@ -1145,6 +1146,22 @@ var config = { }; ``` +### `videoPreference` + +(default `undefined`) + +These settings determine whether HDR video should be selected before SDR video. Which VIDEO-RANGE values are allowed, and in what order of priority can also be specified. + +Format `{ preferHDR: boolean, allowedVideoRanges: ('SDR' | 'PQ' | 'HLG')[] }` + +- Allow all video ranges if `allowedVideoRanges` is unspecified. +- If `preferHDR` is defined, use the value to filter `allowedVideoRanges`. +- Else check window for HDR support and set `preferHDR` to the result. + +When `preferHDR` is set, skip checking if the window supports HDR and instead use the value provided to determine level selection preference via dynamic range. A value of `preferHDR === true` will attempt to use HDR levels before selecting from SDR levels. + +`allowedVideoRanges` can restrict playback to a limited set of VIDEO-RANGE transfer functions and set their priority for selection. For example, to ignore all HDR variants, set `allowedVideoRanges` to `['SDR']`. Or, to ignore all HLG variants, set `allowedVideoRanges` to `['SDR', 'PQ']`. To prioritize PQ variants over HLG, set `allowedVideoRanges` to `['SDR', 'HLG', 'PQ']`. + ### `audioPreference` (default: `undefined`) diff --git a/src/config.ts b/src/config.ts index f9be9ab4483..05b3e52938a 100644 --- a/src/config.ts +++ b/src/config.ts @@ -32,6 +32,7 @@ import type { import type { AudioSelectionOption, SubtitleSelectionOption, + VideoSelectionOption, } from './types/media-playlist'; export type ABRControllerConfig = { @@ -223,6 +224,7 @@ export type StreamControllerConfig = { }; export type SelectionPreferences = { + videoPreference?: VideoSelectionOption; audioPreference?: AudioSelectionOption; subtitlePreference?: SubtitleSelectionOption; }; diff --git a/src/controller/abr-controller.ts b/src/controller/abr-controller.ts index 3747ecd3624..3ee7ff72053 100644 --- a/src/controller/abr-controller.ts +++ b/src/controller/abr-controller.ts @@ -633,7 +633,8 @@ class AbrController implements AbrComponentAPI { let currentCodecSet: string | undefined; let currentVideoRange: VideoRange | undefined = 'SDR'; let currentFrameRate = level?.frameRate || 0; - const audioPreference = config.audioPreference; + + const { audioPreference, videoPreference } = config; const audioTracksByGroup = this.audioTracksByGroup || (this.audioTracksByGroup = getAudioTracksByGroup(allAudioTracks)); @@ -654,10 +655,14 @@ class AbrController implements AbrComponentAPI { currentVideoRange, currentBw, audioPreference, + videoPreference, ); - const { codecSet, videoRange, minFramerate, minBitrate } = startTier; + const { codecSet, videoRanges, minFramerate, minBitrate, preferHDR } = + startTier; currentCodecSet = codecSet; - currentVideoRange = videoRange; + currentVideoRange = preferHDR + ? videoRanges[videoRanges.length - 1] + : videoRanges[0]; currentFrameRate = minFramerate; currentBw = Math.max(currentBw, minBitrate); logger.log(`[abr] picked start tier ${JSON.stringify(startTier)}`); diff --git a/src/hls.ts b/src/hls.ts index d0b6bf666ec..f3c37961e22 100644 --- a/src/hls.ts +++ b/src/hls.ts @@ -30,6 +30,7 @@ import type { AudioSelectionOption, MediaPlaylist, SubtitleSelectionOption, + VideoSelectionOption, } from './types/media-playlist'; import type { HlsConfig } from './config'; import type { BufferInfo } from './utils/buffer-helper'; @@ -959,6 +960,7 @@ export default class Hls implements HlsEventEmitter { export type { AudioSelectionOption, SubtitleSelectionOption, + VideoSelectionOption, MediaPlaylist, ErrorDetails, ErrorTypes, diff --git a/src/types/media-playlist.ts b/src/types/media-playlist.ts index e4870535b1b..5bb510ac3c7 100644 --- a/src/types/media-playlist.ts +++ b/src/types/media-playlist.ts @@ -1,5 +1,6 @@ import type { AttrList } from '../utils/attr-list'; import type { LevelDetails } from '../loader/level-details'; +import type { VideoRange } from './level'; export type AudioPlaylistType = 'AUDIO'; @@ -9,6 +10,11 @@ export type SubtitlePlaylistType = 'SUBTITLES' | 'CLOSED-CAPTIONS'; export type MediaPlaylistType = MainPlaylistType | SubtitlePlaylistType; +export type VideoSelectionOption = { + preferHDR?: boolean; + allowedVideoRanges?: Array; +}; + export type AudioSelectionOption = { lang?: string; assocLang?: string; diff --git a/src/utils/hdr.ts b/src/utils/hdr.ts new file mode 100644 index 00000000000..3d9549ac134 --- /dev/null +++ b/src/utils/hdr.ts @@ -0,0 +1,70 @@ +import { type VideoRange, VideoRangeValues } from '../types/level'; +import type { VideoSelectionOption } from '../types/media-playlist'; + +/** + * @returns Whether we can detect and validate HDR capability within the window context + */ +export function isHdrSupported() { + if (typeof matchMedia === 'function') { + const mediaQueryList = matchMedia('(dynamic-range: high)'); + const badQuery = matchMedia('bad query'); + if (mediaQueryList.media !== badQuery.media) { + return mediaQueryList.matches === true; + } + } + return false; +} + +/** + * Sanitizes inputs to return the active video selection options for HDR/SDR. + * When both inputs are null: + * + * `{ preferHDR: false, allowedVideoRanges: [] }` + * + * When `currentVideoRange` non-null, maintain the active range: + * + * `{ preferHDR: currentVideoRange !== 'SDR', allowedVideoRanges: [currentVideoRange] }` + * + * When VideoSelectionOption non-null: + * + * - Allow all video ranges if `allowedVideoRanges` unspecified. + * - If `preferHDR` is non-null use the value to filter `allowedVideoRanges`. + * - Else check window for HDR support and set `preferHDR` to the result. + * + * @param currentVideoRange + * @param videoPreference + */ +export function getVideoSelectionOptions( + currentVideoRange: VideoRange | undefined, + videoPreference: VideoSelectionOption | undefined, +) { + let preferHDR = false; + let allowedVideoRanges: Array = []; + + if (currentVideoRange) { + preferHDR = currentVideoRange !== 'SDR'; + allowedVideoRanges = [currentVideoRange]; + } + + if (videoPreference) { + allowedVideoRanges = + videoPreference.allowedVideoRanges || VideoRangeValues.slice(0); + preferHDR = + videoPreference.preferHDR !== undefined + ? videoPreference.preferHDR + : isHdrSupported(); + + if (preferHDR) { + allowedVideoRanges = allowedVideoRanges.filter( + (range: VideoRange) => range !== 'SDR', + ); + } else { + allowedVideoRanges = ['SDR']; + } + } + + return { + preferHDR, + allowedVideoRanges, + }; +} diff --git a/src/utils/rendition-helper.ts b/src/utils/rendition-helper.ts index 2191057cb8f..8f0fdbe213c 100644 --- a/src/utils/rendition-helper.ts +++ b/src/utils/rendition-helper.ts @@ -1,10 +1,12 @@ import { codecsSetSelectionPreferenceValue } from './codecs'; +import { getVideoSelectionOptions } from './hdr'; import { logger } from './logger'; import type { Level, VideoRange } from '../types/level'; import type { AudioSelectionOption, MediaPlaylist, SubtitleSelectionOption, + VideoSelectionOption, } from '../types/media-playlist'; export type CodecSetTier = { @@ -26,16 +28,18 @@ type AudioTrackGroup = { }; type StartParameters = { codecSet: string | undefined; - videoRange: VideoRange | undefined; + videoRanges: Array; + preferHDR: boolean; minFramerate: number; minBitrate: number; }; export function getStartCodecTier( codecTiers: Record, - videoRange: VideoRange | undefined, + currentVideoRange: VideoRange | undefined, currentBw: number, audioPreference: AudioSelectionOption | undefined, + videoPreference: VideoSelectionOption | undefined, ): StartParameters { const codecSets = Object.keys(codecTiers); const channelsPreference = audioPreference?.channels; @@ -48,14 +52,25 @@ export function getStartCodecTier( let minFramerate = Infinity; let minBitrate = Infinity; let selectedScore = 0; + let videoRanges: Array = []; + + const { preferHDR, allowedVideoRanges } = getVideoSelectionOptions( + currentVideoRange, + videoPreference, + ); + for (let i = codecSets.length; i--; ) { const tier = codecTiers[codecSets[i]]; hasStereo = tier.channels[2] > 0; minHeight = Math.min(minHeight, tier.minHeight); minFramerate = Math.min(minFramerate, tier.minFramerate); minBitrate = Math.min(minBitrate, tier.minBitrate); - if (videoRange) { - hasCurrentVideoRange ||= tier.videoRanges[videoRange] > 0; + const matchingVideoRanges = allowedVideoRanges.filter( + (range) => tier.videoRanges[range] > 0, + ); + if (matchingVideoRanges.length > 0) { + hasCurrentVideoRange = true; + videoRanges = matchingVideoRanges; } } minHeight = Number.isFinite(minHeight) ? minHeight : 0; @@ -64,9 +79,10 @@ export function getStartCodecTier( const maxFramerate = Math.max(30, minFramerate); minBitrate = Number.isFinite(minBitrate) ? minBitrate : currentBw; currentBw = Math.max(minBitrate, currentBw); - // If there are no SDR variants, set currentVideoRange to undefined + // If there are no variants with matching preference, set currentVideoRange to undefined if (!hasCurrentVideoRange) { - videoRange = undefined; + currentVideoRange = undefined; + videoRanges = []; } const codecSet = codecSets.reduce( (selected: string | undefined, candidate: string) => { @@ -134,10 +150,12 @@ export function getStartCodecTier( ); return selected; } - if (videoRange && candidateTier.videoRanges[videoRange] === 0) { + if (!videoRanges.some((range) => candidateTier.videoRanges[range] > 0)) { logStartCodecCandidateIgnored( candidate, - `no variants with VIDEO-RANGE of ${videoRange} found`, + `no variants with VIDEO-RANGE of ${JSON.stringify( + videoRanges, + )} found`, ); return selected; } @@ -164,7 +182,8 @@ export function getStartCodecTier( ); return { codecSet, - videoRange, + videoRanges, + preferHDR, minFramerate, minBitrate, }; From a1b505106fa6f8084d12ad4f8dccd6ccecb25e77 Mon Sep 17 00:00:00 2001 From: Rob Walch Date: Wed, 29 Nov 2023 15:20:04 -0800 Subject: [PATCH 2/3] Recover from media error after MediaSource ended following SourceBuffer update error event --- src/controller/buffer-controller.ts | 18 ++++++++++-------- src/controller/error-controller.ts | 5 +++++ .../controller/buffer-controller-operations.ts | 8 ++++++-- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/controller/buffer-controller.ts b/src/controller/buffer-controller.ts index d94c0444389..16d7db0ab32 100755 --- a/src/controller/buffer-controller.ts +++ b/src/controller/buffer-controller.ts @@ -400,8 +400,8 @@ export default class BufferController implements ComponentAPI { }, onStart: () => {}, onComplete: () => {}, - onError: (e) => { - this.warn(`Failed to change ${type} SourceBuffer type`, e); + onError: (error: Error) => { + this.warn(`Failed to change ${type} SourceBuffer type`, error); }, }; @@ -495,7 +495,7 @@ export default class BufferController implements ComponentAPI { timeRanges, }); }, - onError: (err) => { + onError: (error: Error) => { // in case any error occured while appending, put back segment in segments table const event: ErrorData = { type: ErrorTypes.MEDIA_ERROR, @@ -505,12 +505,12 @@ export default class BufferController implements ComponentAPI { frag, part, chunkMeta, - error: err, - err, + error, + err: error, fatal: false, }; - if (err.code === DOMException.QUOTA_EXCEEDED_ERR) { + if ((error as DOMException).code === DOMException.QUOTA_EXCEEDED_ERR) { // QuotaExceededError: http://www.w3.org/TR/html5/infrastructure.html#quotaexceedederror // let's stop appending any segments, and report BUFFER_FULL_ERROR error event.details = ErrorDetails.BUFFER_FULL_ERROR; @@ -1032,7 +1032,9 @@ export default class BufferController implements ComponentAPI { } private _onSBUpdateError(type: SourceBufferName, event: Event) { - const error = new Error(`${type} SourceBuffer error`); + const error = new Error( + `${type} SourceBuffer error. MediaSource readyState: ${this.mediaSource?.readyState}`, + ); this.error(`${error}`, event); // according to http://www.w3.org/TR/media-source/#sourcebuffer-append-error // SourceBuffer errors are not necessarily fatal; if so, the HTMLMediaElement will fire an error event @@ -1046,7 +1048,7 @@ export default class BufferController implements ComponentAPI { // updateend is always fired after error, so we'll allow that to shift the current operation off of the queue const operation = this.operationQueue.current(type); if (operation) { - operation.onError(event); + operation.onError(error); } } diff --git a/src/controller/error-controller.ts b/src/controller/error-controller.ts index a697b3dd953..bc6f8810b95 100644 --- a/src/controller/error-controller.ts +++ b/src/controller/error-controller.ts @@ -453,6 +453,11 @@ export default class ErrorController implements NetworkComponentAPI { data.details !== ErrorDetails.FRAG_GAP ) { data.fatal = true; + } else if (/MediaSource readyState: ended/.test(data.error.message)) { + this.warn( + `MediaSource ended after "${data.sourceBufferName}" sourceBuffer append error. Attempting to recover from media error.`, + ); + this.hls.recoverMediaError(); } break; case NetworkErrorAction.RetryRequest: diff --git a/tests/unit/controller/buffer-controller-operations.ts b/tests/unit/controller/buffer-controller-operations.ts index 1e71357c3f8..a9e1b64c611 100644 --- a/tests/unit/controller/buffer-controller-operations.ts +++ b/tests/unit/controller/buffer-controller-operations.ts @@ -199,6 +199,7 @@ describe('BufferController', function () { currentQueue.push(operation); const errorEvent = new Event('error'); bufferController.sourceBuffer[name].dispatchEvent(errorEvent); + const sbErrorObject = triggerSpy.getCall(0).lastArg.error; expect( onError, @@ -206,8 +207,11 @@ describe('BufferController', function () { ).to.have.callCount(i + 1); expect( onError, - 'onError should be called with the error event', - ).to.have.been.calledWith(errorEvent); + 'onError should be called with an error object', + ).to.have.been.calledWith(sbErrorObject); + expect(sbErrorObject.message).equals( + 'audio SourceBuffer error. MediaSource readyState: open', + ); expect( triggerSpy, 'ERROR should have been triggered in response to the SourceBuffer error', From d0c0aa632f47f46d19d45b74e4845587d0a6a2af Mon Sep 17 00:00:00 2001 From: Rob Walch Date: Thu, 30 Nov 2023 16:17:25 -0800 Subject: [PATCH 3/3] Fix exception on 2019 Tizen where MediaCapabilities is undefined --- src/controller/abr-controller.ts | 6 ++++-- src/utils/mediacapabilities-helper.ts | 8 +++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/controller/abr-controller.ts b/src/controller/abr-controller.ts index 3ee7ff72053..c44f57801b5 100644 --- a/src/controller/abr-controller.ts +++ b/src/controller/abr-controller.ts @@ -691,12 +691,14 @@ class AbrController implements AbrComponentAPI { !levelInfo.supportedResult && !levelInfo.supportedPromise ) { - const mediaCapabilities = navigator.mediaCapabilities; + const mediaCapabilities = navigator.mediaCapabilities as + | MediaCapabilities + | undefined; if ( + typeof mediaCapabilities?.decodingInfo === 'function' && requiresMediaCapabilitiesDecodingInfo( levelInfo, audioTracksByGroup, - mediaCapabilities, currentVideoRange, currentFrameRate, currentBw, diff --git a/src/utils/mediacapabilities-helper.ts b/src/utils/mediacapabilities-helper.ts index 33dcf5b8a74..52758969464 100644 --- a/src/utils/mediacapabilities-helper.ts +++ b/src/utils/mediacapabilities-helper.ts @@ -32,7 +32,6 @@ export const SUPPORTED_INFO_CACHE: Record< export function requiresMediaCapabilitiesDecodingInfo( level: Level, audioTracksByGroup: AudioTracksByGroup, - mediaCapabilities: MediaCapabilities | undefined, currentVideoRange: VideoRange | undefined, currentFrameRate: number, currentBw: number, @@ -75,8 +74,7 @@ export function requiresMediaCapabilitiesDecodingInfo( } } return ( - (typeof mediaCapabilities?.decodingInfo == 'function' && - level.videoCodec !== undefined && + (level.videoCodec !== undefined && ((level.width > 1920 && level.height > 1088) || (level.height > 1920 && level.width > 1088) || level.frameRate > Math.max(currentFrameRate, 30) || @@ -94,11 +92,11 @@ export function requiresMediaCapabilitiesDecodingInfo( export function getMediaDecodingInfoPromise( level: Level, audioTracksByGroup: AudioTracksByGroup, - mediaCapabilities: MediaCapabilities, + mediaCapabilities: MediaCapabilities | undefined, ): Promise { const videoCodecs = level.videoCodec; const audioCodecs = level.audioCodec; - if (!videoCodecs || !audioCodecs) { + if (!videoCodecs || !audioCodecs || !mediaCapabilities) { return Promise.resolve(SUPPORTED_INFO_DEFAULT); }