From 09bdd61f6e47f4239bf5fbc3d60754f61ec5be94 Mon Sep 17 00:00:00 2001 From: lonebyte <61915324+lonebyte@users.noreply.github.com> Date: Tue, 15 Aug 2023 17:35:05 +0200 Subject: [PATCH] fix: Support fLaC and Opus codec strings in HLS (#5454) Fixes: #5453 --- lib/util/manifest_parser_utils.js | 6 ++- lib/util/stream_utils.js | 87 ++++++++++++++++++++++--------- test/hls/hls_parser_unit.js | 72 +++++++++++++++++++++++++ test/util/stream_utils_unit.js | 46 ++++++++++++++++ 4 files changed, 184 insertions(+), 27 deletions(-) diff --git a/lib/util/manifest_parser_utils.js b/lib/util/manifest_parser_utils.js index 122511611f..eabf33e48a 100644 --- a/lib/util/manifest_parser_utils.js +++ b/lib/util/manifest_parser_utils.js @@ -180,8 +180,10 @@ shaka.util.ManifestParserUtils.VIDEO_CODEC_REGEXPS_ = [ */ shaka.util.ManifestParserUtils.AUDIO_CODEC_REGEXPS_ = [ /^vorbis$/, - /^opus$/, - /^flac$/, + /^Opus$/, // correct codec string according to RFC 6381 section 3.3 + /^opus$/, // some manifests wrongfully use this + /^fLaC$/, // correct codec string according to RFC 6381 section 3.3 + /^flac$/, // some manifests wrongfully use this /^mp4a/, /^[ae]c-3$/, /^ac-4$/, diff --git a/lib/util/stream_utils.js b/lib/util/stream_utils.js index 0aec784a3d..51b10267af 100644 --- a/lib/util/stream_utils.js +++ b/lib/util/stream_utils.js @@ -437,48 +437,62 @@ shaka.util.StreamUtils = class { manifest.variants = manifest.variants.filter((variant) => { // See: https://github.com/shaka-project/shaka-player/issues/3860 const video = variant.video; + const ContentType = shaka.util.ManifestParserUtils.ContentType; const Capabilities = shaka.media.Capabilities; + const ManifestParserUtils = shaka.util.ManifestParserUtils; + const MimeUtils = shaka.util.MimeUtils; + const StreamUtils = shaka.util.StreamUtils; + if (video) { - let videoCodecs = - shaka.util.StreamUtils.getCorrectVideoCodecs_(video.codecs); + let videoCodecs = StreamUtils.getCorrectVideoCodecs_(video.codecs); + // For multiplexed streams. Here we must check the audio of the // stream to see if it is compatible. if (video.codecs.includes(',')) { const allCodecs = video.codecs.split(','); - videoCodecs = shaka.util.ManifestParserUtils.guessCodecs( + + videoCodecs = ManifestParserUtils.guessCodecs( ContentType.VIDEO, allCodecs); - videoCodecs = - shaka.util.StreamUtils.getCorrectVideoCodecs_(videoCodecs); - let audioCodecs = shaka.util.ManifestParserUtils.guessCodecs( + videoCodecs = StreamUtils.getCorrectVideoCodecs_(videoCodecs); + + let audioCodecs = ManifestParserUtils.guessCodecs( ContentType.AUDIO, allCodecs); - audioCodecs = - shaka.util.StreamUtils.getCorrectAudioCodecs_(audioCodecs); - const audioFullType = shaka.util.MimeUtils.getFullOrConvertedType( + audioCodecs = StreamUtils.getCorrectAudioCodecs_(audioCodecs); + + const audioFullType = MimeUtils.getFullOrConvertedType( video.mimeType, audioCodecs, ContentType.AUDIO); + if (!Capabilities.isTypeSupported(audioFullType)) { return false; } + // Update the codec string with the (possibly) converted codecs. videoCodecs = [videoCodecs, audioCodecs].join(','); } - const fullType = shaka.util.MimeUtils.getFullOrConvertedType( + + const fullType = MimeUtils.getFullOrConvertedType( video.mimeType, videoCodecs, ContentType.VIDEO); + if (!Capabilities.isTypeSupported(fullType)) { return false; } + // Update the codec string with the (possibly) converted codecs. video.codecs = videoCodecs; } + const audio = variant.audio; + if (audio) { - const codecs = - shaka.util.StreamUtils.getCorrectAudioCodecs_(audio.codecs); - const fullType = shaka.util.MimeUtils.getFullOrConvertedType( + const codecs = StreamUtils.getCorrectAudioCodecs_(audio.codecs); + const fullType = MimeUtils.getFullOrConvertedType( audio.mimeType, codecs, ContentType.AUDIO); + if (!Capabilities.isTypeSupported(fullType)) { return false; } + // Update the codec string with the (possibly) converted codecs. audio.codecs = codecs; } @@ -490,7 +504,7 @@ shaka.util.StreamUtils = class { (video.codecs.includes('avc1.') || video.codecs.includes('avc3.'))) { shaka.log.debug('Dropping variant - not compatible with platform', - shaka.util.StreamUtils.getVariantSummaryString_(variant)); + StreamUtils.getVariantSummaryString_(variant)); return false; } @@ -500,7 +514,7 @@ shaka.util.StreamUtils = class { // Filter out all unsupported variants. if (!supported) { shaka.log.debug('Dropping variant - not compatible with platform', - shaka.util.StreamUtils.getVariantSummaryString_(variant)); + StreamUtils.getVariantSummaryString_(variant)); } return supported; }); @@ -564,7 +578,11 @@ shaka.util.StreamUtils = class { static getDecodingConfigs_(variant, usePersistentLicenses, srcEquals) { const audio = variant.audio; const video = variant.video; + const ContentType = shaka.util.ManifestParserUtils.ContentType; + const ManifestParserUtils = shaka.util.ManifestParserUtils; + const MimeUtils = shaka.util.MimeUtils; + const StreamUtils = shaka.util.StreamUtils; /** @type {!MediaDecodingConfiguration} */ const mediaDecodingConfig = { @@ -573,19 +591,23 @@ shaka.util.StreamUtils = class { if (video) { let videoCodecs = video.codecs; + // For multiplexed streams with audio+video codecs, the config should have // AudioConfiguration and VideoConfiguration. if (video.codecs.includes(',')) { const allCodecs = video.codecs.split(','); - videoCodecs = shaka.util.ManifestParserUtils.guessCodecs( + + videoCodecs = ManifestParserUtils.guessCodecs( ContentType.VIDEO, allCodecs); - videoCodecs = - shaka.util.StreamUtils.getCorrectVideoCodecs_(videoCodecs); - const audioCodecs = shaka.util.ManifestParserUtils.guessCodecs( + videoCodecs = StreamUtils.getCorrectVideoCodecs_(videoCodecs); + + let audioCodecs = ManifestParserUtils.guessCodecs( ContentType.AUDIO, allCodecs); + audioCodecs = StreamUtils.getCorrectAudioCodecs_(audioCodecs); - const audioFullType = shaka.util.MimeUtils.getFullOrConvertedType( + const audioFullType = MimeUtils.getFullOrConvertedType( video.mimeType, audioCodecs, ContentType.AUDIO); + mediaDecodingConfig.audio = { contentType: audioFullType, channels: 2, @@ -594,9 +616,11 @@ shaka.util.StreamUtils = class { spatialRendering: false, }; } - videoCodecs = shaka.util.StreamUtils.getCorrectVideoCodecs_(videoCodecs); - const fullType = shaka.util.MimeUtils.getFullOrConvertedType( + + videoCodecs = StreamUtils.getCorrectVideoCodecs_(videoCodecs); + const fullType = MimeUtils.getFullOrConvertedType( video.mimeType, videoCodecs, ContentType.VIDEO); + // VideoConfiguration mediaDecodingConfig.video = { contentType: fullType, @@ -629,9 +653,8 @@ shaka.util.StreamUtils = class { } } if (audio) { - const codecs = - shaka.util.StreamUtils.getCorrectAudioCodecs_(audio.codecs); - const fullType = shaka.util.MimeUtils.getFullOrConvertedType( + const codecs = StreamUtils.getCorrectAudioCodecs_(audio.codecs); + const fullType = MimeUtils.getFullOrConvertedType( audio.mimeType, codecs, ContentType.AUDIO); // AudioConfiguration @@ -762,6 +785,20 @@ shaka.util.StreamUtils = class { * @private */ static getCorrectAudioCodecs_(codecs) { + // According to RFC 6381 section 3.3, 'fLaC' is actually the correct + // codec string. We still need to map it to 'flac', as some browsers + // currently don't support 'fLaC', while 'flac' is supported by most + // major browsers. + // See https://bugs.chromium.org/p/chromium/issues/detail?id=1422728 + if (codecs === 'fLaC') { + return 'flac'; + } + + // The same is true for 'Opus'. + if (codecs === 'Opus') { + return 'opus'; + } + // Some Tizen devices seem to misreport AC-3 support, but correctly // report EC-3 support. So query EC-3 as a fallback for AC-3. // See https://github.com/shaka-project/shaka-player/issues/2989 for diff --git a/test/hls/hls_parser_unit.js b/test/hls/hls_parser_unit.js index bee4ba9862..1978d0512f 100644 --- a/test/hls/hls_parser_unit.js +++ b/test/hls/hls_parser_unit.js @@ -628,6 +628,78 @@ describe('HlsParser', () => { await testHlsParser(master, media, manifest); }); + it('accepts fLaC codec as audio/mp4', async () => { + const master = [ + '#EXTM3U\n', + '#EXT-X-STREAM-INF:BANDWIDTH=1000000,CODECS="fLaC"\n', + 'audio\n', + '#EXT-X-STREAM-INF:BANDWIDTH=1000000,CODECS="flac"\n', + 'audio2\n', + ].join(''); + + const media = [ + '#EXTM3U\n', + '#EXT-X-PLAYLIST-TYPE:VOD\n', + '#EXT-X-MAP:URI="init.mp4",BYTERANGE="616@0"\n', + '#EXTINF:5,\n', + '#EXT-X-BYTERANGE:121090@616\n', + 'main.mp4', + ].join(''); + + const manifest = shaka.test.ManifestGenerator.generate((manifest) => { + manifest.anyTimeline(); + manifest.addPartialVariant((variant) => { + variant.addPartialStream(ContentType.AUDIO, (stream) => { + stream.mime('audio/mp4', 'fLaC'); + }); + }); + manifest.addPartialVariant((variant) => { + variant.addPartialStream(ContentType.AUDIO, (stream) => { + stream.mime('audio/mp4', 'flac'); + }); + }); + manifest.sequenceMode = true; + }); + + await testHlsParser(master, media, manifest); + }); + + it('accepts Opus codec as audio/mp4', async () => { + const master = [ + '#EXTM3U\n', + '#EXT-X-STREAM-INF:BANDWIDTH=128000,CODECS="Opus"\n', + 'audio\n', + '#EXT-X-STREAM-INF:BANDWIDTH=128000,CODECS="opus"\n', + 'audio2\n', + ].join(''); + + const media = [ + '#EXTM3U\n', + '#EXT-X-PLAYLIST-TYPE:VOD\n', + '#EXT-X-MAP:URI="init.mp4",BYTERANGE="616@0"\n', + '#EXTINF:5,\n', + '#EXT-X-BYTERANGE:121090@616\n', + 'main.mp4', + ].join(''); + + const manifest = shaka.test.ManifestGenerator.generate((manifest) => { + manifest.anyTimeline(); + manifest.addPartialVariant((variant) => { + variant.addPartialStream(ContentType.AUDIO, (stream) => { + stream.mime('audio/mp4', 'Opus'); + }); + }); + manifest.addPartialVariant((variant) => { + variant.addPartialStream(ContentType.AUDIO, (stream) => { + stream.mime('audio/mp4', 'opus'); + }); + }); + manifest.sequenceMode = true; + }); + + await testHlsParser(master, media, manifest); + }); + it('parses audio+video variant with closed captions', async () => { const master = [ '#EXTM3U\n', diff --git a/test/util/stream_utils_unit.js b/test/util/stream_utils_unit.js index da02e5a5e0..6d8b587cab 100644 --- a/test/util/stream_utils_unit.js +++ b/test/util/stream_utils_unit.js @@ -718,6 +718,52 @@ describe('StreamUtils', () => { expect(manifest.variants.length).toBe(1); }); + it('supports fLaC codec', async () => { + if (!MediaSource.isTypeSupported('audio/mp4; codecs="flac"')) { + pending('Codec fLaC is not supported by the platform.'); + } + manifest = shaka.test.ManifestGenerator.generate((manifest) => { + manifest.addVariant(0, (variant) => { + variant.addAudio(1, (stream) => { + stream.mime('audio/mp4', 'fLaC'); + }); + }); + manifest.addVariant(2, (variant) => { + variant.addAudio(3, (stream) => { + stream.mime('audio/mp4', 'flac'); + }); + }); + }); + + await shaka.util.StreamUtils.filterManifest( + fakeDrmEngine, /* currentVariant= */ null, manifest); + + expect(manifest.variants.length).toBe(2); + }); + + it('supports Opus codec', async () => { + if (!MediaSource.isTypeSupported('audio/mp4; codecs="opus"')) { + pending('Codec Opus is not supported by the platform.'); + } + manifest = shaka.test.ManifestGenerator.generate((manifest) => { + manifest.addVariant(0, (variant) => { + variant.addAudio(1, (stream) => { + stream.mime('audio/mp4', 'Opus'); + }); + }); + manifest.addVariant(2, (variant) => { + variant.addAudio(3, (stream) => { + stream.mime('audio/mp4', 'opus'); + }); + }); + }); + + await shaka.util.StreamUtils.filterManifest( + fakeDrmEngine, /* currentVariant= */ null, manifest); + + expect(manifest.variants.length).toBe(2); + }); + it('supports legacy AVC1 codec', async () => { if (!MediaSource.isTypeSupported('video/mp4; codecs="avc1.42001e"')) { pending('Codec avc1.42001e is not supported by the platform.');