From da8d3b21b3945c51a44750d5007baea9f76d3409 Mon Sep 17 00:00:00 2001 From: Alvaro Velad Galvan Date: Tue, 2 May 2023 19:14:54 +0200 Subject: [PATCH] fix: Transmux containerless to the correct mimetype --- lib/hls/hls_parser.js | 18 ----------- lib/media/media_source_engine.js | 38 ++++++++++------------ lib/transmuxer/muxjs_transmuxer.js | 12 +++++-- lib/util/mime_utils.js | 51 +++++++++++++++++++++++++++--- test/hls/hls_parser_unit.js | 12 ++++--- test/player_integration.js | 11 +++++++ 6 files changed, 89 insertions(+), 53 deletions(-) diff --git a/lib/hls/hls_parser.js b/lib/hls/hls_parser.js index c0abb127c6..0314e5a975 100644 --- a/lib/hls/hls_parser.js +++ b/lib/hls/hls_parser.js @@ -18,7 +18,6 @@ goog.require('shaka.log'); goog.require('shaka.media.DrmEngine'); goog.require('shaka.media.InitSegmentReference'); goog.require('shaka.media.ManifestParser'); -goog.require('shaka.media.MediaSourceEngine'); goog.require('shaka.media.PresentationTimeline'); goog.require('shaka.media.SegmentIndex'); goog.require('shaka.media.SegmentReference'); @@ -1918,9 +1917,6 @@ shaka.hls.HlsParser = class { // segment index, then download when createSegmentIndex is called. const stream = this.makeStreamObject_(codecs, type, language, primary, name, channelsCount, closedCaptions, characteristics, forced, spatialAudio); - if (shaka.media.MediaSourceEngine.RAW_FORMATS.includes(stream.mimeType)) { - stream.codecs = ''; - } const streamInfo = { stream, type, @@ -1983,9 +1979,6 @@ shaka.hls.HlsParser = class { stream.drmInfos = realStream.drmInfos; stream.keyIds = realStream.keyIds; stream.mimeType = realStream.mimeType; - if (shaka.media.MediaSourceEngine.RAW_FORMATS.includes(stream.mimeType)) { - stream.codecs = ''; - } // Since we lazy-loaded this content, the player may need to create new // sessions for the DRM info in this stream. @@ -2000,10 +1993,6 @@ shaka.hls.HlsParser = class { // To aid manifest filtering, assume before loading that all video // renditions have the same MIME type. (And likewise for audio.) otherStreamInfo.stream.mimeType = realStream.mimeType; - if (shaka.media.MediaSourceEngine.RAW_FORMATS - .includes(otherStreamInfo.stream.mimeType)) { - otherStreamInfo.stream.codecs = ''; - } } } } @@ -2134,13 +2123,6 @@ shaka.hls.HlsParser = class { streamInfo.stream.segmentIndex.fit(/* periodStart= */ 0, minDuration); } } - // MediaSource expects no codec strings combined with raw formats. - for (const streamInfo of streamInfos) { - const stream = streamInfo.stream; - if (shaka.media.MediaSourceEngine.RAW_FORMATS.includes(stream.mimeType)) { - stream.codecs = ''; - } - } this.notifySegmentsForStreams_(streamInfos.map((s) => s.stream)); if (this.config_.hls.ignoreManifestProgramDateTime) { this.syncStreamsWithSequenceNumber_(streamInfos); diff --git a/lib/media/media_source_engine.js b/lib/media/media_source_engine.js index 093ce79d24..06dedae02b 100644 --- a/lib/media/media_source_engine.js +++ b/lib/media/media_source_engine.js @@ -181,13 +181,15 @@ shaka.media.MediaSourceEngine = class { * @return {boolean} */ static isStreamSupported(stream) { - const fullMimeType = shaka.util.MimeUtils.getFullType( + const MimeUtils = shaka.util.MimeUtils; + const fullMimeType = MimeUtils.getFullType(stream.mimeType, stream.codecs); + const extendedMimeType = MimeUtils.getExtendedType(stream); + const fullMimeTypeWithAllCodecs = MimeUtils.getFullTypeWithAllCodecs( stream.mimeType, stream.codecs); - const extendedMimeType = shaka.util.MimeUtils.getExtendedType(stream); const TransmuxerEngine = shaka.transmuxer.TransmuxerEngine; return shaka.text.TextEngine.isTypeSupported(fullMimeType) || shaka.media.Capabilities.isTypeSupported(extendedMimeType) || - TransmuxerEngine.isSupported(fullMimeType, stream.type); + TransmuxerEngine.isSupported(fullMimeTypeWithAllCodecs, stream.type); } /** @@ -235,7 +237,7 @@ shaka.media.MediaSourceEngine = class { 'application/ttml+xml', 'application/mp4; codecs="stpp"', // Containerless types - ...shaka.media.MediaSourceEngine.RAW_FORMATS, + ...shaka.util.MimeUtils.RAW_FORMATS, ]; const support = {}; @@ -391,17 +393,22 @@ shaka.media.MediaSourceEngine = class { let needTransmux = this.config_.forceTransmux; if (!shaka.media.Capabilities.isTypeSupported(mimeType) || (!sequenceMode && - shaka.media.MediaSourceEngine.RAW_FORMATS.includes(mimeType))) { + shaka.util.MimeUtils.RAW_FORMATS.includes(mimeType))) { needTransmux = true; } + const mimeTypeWithAllCodecs = + shaka.util.MimeUtils.getFullTypeWithAllCodecs( + stream.mimeType, stream.codecs); const TransmuxerEngine = shaka.transmuxer.TransmuxerEngine; if (needTransmux && - TransmuxerEngine.isSupported(mimeType, contentType)) { - const transmuxerPlugin = TransmuxerEngine.findTransmuxer(mimeType); + TransmuxerEngine.isSupported(mimeTypeWithAllCodecs, contentType)) { + const transmuxerPlugin = + TransmuxerEngine.findTransmuxer(mimeTypeWithAllCodecs); if (transmuxerPlugin) { const transmuxer = transmuxerPlugin(); this.transmuxers_[contentType] = transmuxer; - mimeType = transmuxer.convertCodecs(contentType, mimeType); + mimeType = + transmuxer.convertCodecs(contentType, mimeTypeWithAllCodecs); } } const type = mimeType + this.config_.sourceBufferExtraFeatures; @@ -639,7 +646,7 @@ shaka.media.MediaSourceEngine = class { if (this.transmuxers_[contentType]) { mimeType = this.transmuxers_[contentType].getOrginalMimeType(); } - if (shaka.media.MediaSourceEngine.RAW_FORMATS.includes(mimeType)) { + if (shaka.util.MimeUtils.RAW_FORMATS.includes(mimeType)) { const frames = shaka.util.Id3Utils.getID3Frames(uint8ArrayData); if (frames.length && reference) { if (attemptTimestampOffsetCalculation) { @@ -1547,16 +1554,3 @@ shaka.media.MediaSourceEngine.SourceBufferMode_ = { SEQUENCE: 'sequence', SEGMENTS: 'segments', }; - - -/** - * MIME types of raw formats. - * - * @const {!Array.} - */ -shaka.media.MediaSourceEngine.RAW_FORMATS = [ - 'audio/aac', - 'audio/ac3', - 'audio/ec3', - 'audio/mpeg', -]; diff --git a/lib/transmuxer/muxjs_transmuxer.js b/lib/transmuxer/muxjs_transmuxer.js index 1feee6f695..37fa23ace3 100644 --- a/lib/transmuxer/muxjs_transmuxer.js +++ b/lib/transmuxer/muxjs_transmuxer.js @@ -13,6 +13,7 @@ goog.require('shaka.transmuxer.TransmuxerEngine'); goog.require('shaka.util.BufferUtils'); goog.require('shaka.util.Error'); goog.require('shaka.util.ManifestParserUtils'); +goog.require('shaka.util.MimeUtils'); goog.require('shaka.util.PublicPromise'); goog.require('shaka.util.Uint8ArrayUtils'); @@ -85,7 +86,7 @@ shaka.transmuxer.MuxjsTransmuxer = class { } if (isAac) { - return Capabilities.isTypeSupported(this.convertAacCodecs_()); + return Capabilities.isTypeSupported(this.convertAacCodecs_(mimeType)); } if (contentType) { @@ -130,7 +131,7 @@ shaka.transmuxer.MuxjsTransmuxer = class { */ convertCodecs(contentType, mimeType) { if (this.isAacContainer_(mimeType)) { - return this.convertAacCodecs_(); + return this.convertAacCodecs_(mimeType); } else if (this.isTsContainer_(mimeType)) { return this.convertTsCodecs_(contentType, mimeType); } @@ -140,10 +141,15 @@ shaka.transmuxer.MuxjsTransmuxer = class { /** * For aac stream, convert its codecs to MP4 codecs. + * @param {string} mimeType * @return {string} * @private */ - convertAacCodecs_() { + convertAacCodecs_(mimeType) { + const codecs = shaka.util.MimeUtils.getCodecs(mimeType); + if (codecs != '') { + return `audio/mp4; codecs="${codecs}"`; + } return 'audio/mp4; codecs="mp4a.40.2"'; } diff --git a/lib/util/mime_utils.js b/lib/util/mime_utils.js index 0a49fbddfa..f441452e4e 100644 --- a/lib/util/mime_utils.js +++ b/lib/util/mime_utils.js @@ -16,7 +16,7 @@ goog.require('shaka.util.ManifestParserUtils'); shaka.util.MimeUtils = class { /** * Takes a MIME type and optional codecs string and produces the full MIME - * type. + * type. Also remove the codecs for raw formats. * * @param {string} mimeType * @param {string=} codecs @@ -24,6 +24,27 @@ shaka.util.MimeUtils = class { * @export */ static getFullType(mimeType, codecs) { + let fullMimeType = mimeType; + if (codecs) { + if (!shaka.util.MimeUtils.RAW_FORMATS.includes(mimeType)) { + fullMimeType += '; codecs="' + codecs + '"'; + } else { + fullMimeType += '; codecs=""'; + } + } + return fullMimeType; + } + + /** + * Takes a MIME type and optional codecs string and produces the full MIME + * type. + * + * @param {string} mimeType + * @param {string=} codecs + * @return {string} + * @export + */ + static getFullTypeWithAllCodecs(mimeType, codecs) { let fullMimeType = mimeType; if (codecs) { fullMimeType += '; codecs="' + codecs + '"'; @@ -43,12 +64,16 @@ shaka.util.MimeUtils = class { * @return {string} */ static getFullOrConvertedType(mimeType, codecs, contentType) { - const fullMimeType = shaka.util.MimeUtils.getFullType(mimeType, codecs); + const MimeUtils = shaka.util.MimeUtils; + const fullMimeType = MimeUtils.getFullType(mimeType, codecs); + const fullMimeTypeWithAllCodecs = MimeUtils.getFullTypeWithAllCodecs( + mimeType, codecs); const ContentType = shaka.util.ManifestParserUtils.ContentType; const TransmuxerEngine = shaka.transmuxer.TransmuxerEngine; - if (TransmuxerEngine.isSupported(fullMimeType, contentType)) { - return TransmuxerEngine.convertCodecs(contentType, fullMimeType); + if (TransmuxerEngine.isSupported(fullMimeTypeWithAllCodecs, contentType)) { + return TransmuxerEngine.convertCodecs( + contentType, fullMimeTypeWithAllCodecs); } else if (mimeType != 'video/mp2t' && contentType == ContentType.AUDIO) { // video/mp2t is the correct mime type for TS audio, so only replace the // word "video" with "audio" for non-TS audio content. @@ -71,7 +96,11 @@ shaka.util.MimeUtils = class { const extendedMimeParams = shaka.util.MimeUtils.EXTENDED_MIME_PARAMETERS_; extendedMimeParams.forEach((mimeKey, streamKey) => { const value = stream[streamKey]; - if (value) { + if (streamKey == 'codecs' && + shaka.util.MimeUtils.RAW_FORMATS.includes(stream.mimeType)) { + // Skip codecs for raw formats + components.push(mimeKey + '=""'); + } else if (value) { components.push(mimeKey + '="' + value + '"'); } }); @@ -254,3 +283,15 @@ shaka.util.MimeUtils.CEA608_CLOSED_CAPTION_MIMETYPE = 'application/cea-608'; * @const {string} */ shaka.util.MimeUtils.CEA708_CLOSED_CAPTION_MIMETYPE = 'application/cea-708'; + +/** + * MIME types of raw formats. + * + * @const {!Array.} + */ +shaka.util.MimeUtils.RAW_FORMATS = [ + 'audio/aac', + 'audio/ac3', + 'audio/ec3', + 'audio/mpeg', +]; diff --git a/test/hls/hls_parser_unit.js b/test/hls/hls_parser_unit.js index 3eab6a30d8..3d769b9829 100644 --- a/test/hls/hls_parser_unit.js +++ b/test/hls/hls_parser_unit.js @@ -560,7 +560,7 @@ describe('HlsParser', () => { stream.mime('video/mp4', 'avc1'); }); variant.addPartialStream(ContentType.AUDIO, (stream) => { - stream.mime('audio/mp4', ''); + stream.mime('audio/mp4', 'mp4a.40.34'); }); }); manifest.sequenceMode = sequenceMode; @@ -589,7 +589,7 @@ describe('HlsParser', () => { manifest.anyTimeline(); manifest.addPartialVariant((variant) => { variant.addPartialStream(ContentType.AUDIO, (stream) => { - stream.mime('audio/aac', ''); + stream.mime('audio/aac', 'mp4a.40.2'); }); }); manifest.sequenceMode = sequenceMode; @@ -617,7 +617,7 @@ describe('HlsParser', () => { manifest.anyTimeline(); manifest.addPartialVariant((variant) => { variant.addPartialStream(ContentType.AUDIO, (stream) => { - stream.mime('audio/mpeg', ''); + stream.mime('audio/mpeg', 'mp4a.40.34'); }); }); manifest.sequenceMode = sequenceMode; @@ -4618,7 +4618,9 @@ describe('HlsParser', () => { // Before loading, all MIME types agree, and are defaulted to audio/mp4. expect(actualAudio0.mimeType).toBe('audio/mp4'); + expect(actualAudio0.codecs).toBe('mp4a'); expect(actualAudio1.mimeType).toBe('audio/mp4'); + expect(actualAudio1.codecs).toBe('mp4a'); await actualAudio0.createSegmentIndex(); @@ -4627,9 +4629,9 @@ describe('HlsParser', () => { // This is how we avoid having the unloaded tracks filtered out during // startup. expect(actualAudio0.mimeType).toBe('audio/aac'); - expect(actualAudio0.codecs).toBe(''); + expect(actualAudio0.codecs).toBe('mp4a'); expect(actualAudio1.mimeType).toBe('audio/aac'); - expect(actualAudio1.codecs).toBe(''); + expect(actualAudio1.codecs).toBe('mp4a'); }); it('parses media playlists directly', async () => { diff --git a/test/player_integration.js b/test/player_integration.js index e0a3832bd1..94c5eb2769 100644 --- a/test/player_integration.js +++ b/test/player_integration.js @@ -592,6 +592,17 @@ describe('Player', () => { await waiter.waitUntilPlayheadReachesOrFailOnTimeout(video, 1, 10); }); + it('with containerless formats', async () => { + player = new compiledShaka.Player(video); + // eslint-disable-next-line max-len + const url = 'https://storage.googleapis.com/shaka-demo-assets/raw-hls-audio-only/manifest.m3u8'; + await player.load(url, 0); + + // Ensure the video plays. + video.play(); + await waiter.waitUntilPlayheadReachesOrFailOnTimeout(video, 5, 15); + }); + /** * Gets the language of the active Variant. * @return {string}