diff --git a/build/types/transmuxer b/build/types/transmuxer index f899f05f1a..3a14b6ce9f 100644 --- a/build/types/transmuxer +++ b/build/types/transmuxer @@ -1,3 +1,5 @@ # Optional plugins related to transmuxer. ++../../lib/transmuxer/mp3_transmuxer.js ++../../lib/transmuxer/mpeg_audio.js +../../lib/transmuxer/muxjs_transmuxer.js diff --git a/docs/tutorials/upgrade.md b/docs/tutorials/upgrade.md index c72421f4d4..51eb6aa8be 100644 --- a/docs/tutorials/upgrade.md +++ b/docs/tutorials/upgrade.md @@ -98,4 +98,4 @@ application: (deprecated in v4.3.0) - Plugin changes: - - `Transmuxer` plugins now has two new parameters in `transmux()` method. + - `Transmuxer` plugins now has three new parameters in `transmux()` method. diff --git a/externs/shaka/transmuxer.js b/externs/shaka/transmuxer.js index c61c434b8a..77c57308ac 100644 --- a/externs/shaka/transmuxer.js +++ b/externs/shaka/transmuxer.js @@ -45,9 +45,10 @@ shaka.extern.Transmuxer = class { * @param {shaka.extern.Stream} stream * @param {?shaka.media.SegmentReference} reference The segment reference, or * null for init segments + * @param {number} duration * @return {!Promise.} */ - transmux(data, stream, reference) {} + transmux(data, stream, reference, duration) {} }; diff --git a/lib/media/media_source_engine.js b/lib/media/media_source_engine.js index 06dedae02b..d3c8dcf755 100644 --- a/lib/media/media_source_engine.js +++ b/lib/media/media_source_engine.js @@ -771,7 +771,7 @@ shaka.media.MediaSourceEngine = class { if (this.transmuxers_[contentType]) { data = await this.transmuxers_[contentType].transmux( - data, stream, reference); + data, stream, reference, this.mediaSource_.duration); } data = this.workAroundBrokenPlatforms_( diff --git a/lib/transmuxer/mp3_transmuxer.js b/lib/transmuxer/mp3_transmuxer.js new file mode 100644 index 0000000000..c5d3b18bb1 --- /dev/null +++ b/lib/transmuxer/mp3_transmuxer.js @@ -0,0 +1,195 @@ +/*! @license + * Shaka Player + * Copyright 2016 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +goog.provide('shaka.transmuxer.Mp3Transmuxer'); + +goog.require('shaka.media.Capabilities'); +goog.require('shaka.transmuxer.MpegAudio'); +goog.require('shaka.transmuxer.TransmuxerEngine'); +goog.require('shaka.util.BufferUtils'); +goog.require('shaka.util.Error'); +goog.require('shaka.util.Id3Utils'); +goog.require('shaka.util.ManifestParserUtils'); +goog.require('shaka.util.Mp4Generator'); +goog.require('shaka.util.Uint8ArrayUtils'); + + +/** + * @implements {shaka.extern.Transmuxer} + * @export + */ +shaka.transmuxer.Mp3Transmuxer = class { + /** + * @param {string} mimeType + */ + constructor(mimeType) { + /** @private {string} */ + this.originalMimeType_ = mimeType; + + /** @private {number} */ + this.frameIndex_ = 0; + + /** @private {?Uint8Array} */ + this.initSegment = null; + } + + + /** + * @override + * @export + */ + destroy() { + // Nothing + } + + + /** + * Check if the mime type and the content type is supported. + * @param {string} mimeType + * @param {string=} contentType + * @return {boolean} + * @override + * @export + */ + isSupported(mimeType, contentType) { + const Capabilities = shaka.media.Capabilities; + + if (!this.isMpegContainer_(mimeType)) { + return false; + } + const ContentType = shaka.util.ManifestParserUtils.ContentType; + return Capabilities.isTypeSupported( + this.convertCodecs(ContentType.AUDIO, mimeType)); + } + + + /** + * Check if the mimetype is 'audio/mpeg'. + * @param {string} mimeType + * @return {boolean} + * @private + */ + isMpegContainer_(mimeType) { + return mimeType.toLowerCase().split(';')[0] == 'audio/mpeg'; + } + + + /** + * @override + * @export + */ + convertCodecs(contentType, mimeType) { + if (this.isMpegContainer_(mimeType)) { + return 'audio/mp4; codecs="mp3"'; + } + return mimeType; + } + + + /** + * @override + * @export + */ + getOrginalMimeType() { + return this.originalMimeType_; + } + + + /** + * @override + * @export + */ + transmux(data, stream, reference, duration) { + const MpegAudio = shaka.transmuxer.MpegAudio; + const Uint8ArrayUtils = shaka.util.Uint8ArrayUtils; + + const uint8ArrayData = shaka.util.BufferUtils.toUint8(data); + + const id3Data = shaka.util.Id3Utils.getID3Data(uint8ArrayData); + let offset = id3Data.length; + for (; offset < uint8ArrayData.length; offset++) { + if (MpegAudio.probe(uint8ArrayData, offset)) { + break; + } + } + + const timescale = 90000; + let firstHeader; + + /** @type {!Array.} */ + const samples = []; + + while (offset < uint8ArrayData.length) { + const header = MpegAudio.parseHeader(uint8ArrayData, offset); + if (!header) { + return Promise.reject(new shaka.util.Error( + shaka.util.Error.Severity.CRITICAL, + shaka.util.Error.Category.MEDIA, + shaka.util.Error.Code.TRANSMUXING_FAILED)); + } + if (!firstHeader) { + firstHeader = header; + } + if (offset + header.frameLength <= uint8ArrayData.length) { + samples.push({ + data: uint8ArrayData.subarray(offset, offset + header.frameLength), + size: header.frameLength, + duration: MpegAudio.MPEG_AUDIO_SAMPLE_PER_FRAME, + cts: 0, + flags: { + isLeading: 0, + isDependedOn: 0, + hasRedundancy: 0, + degradPrio: 0, + dependsOn: 2, + isNonSync: 0, + }, + }); + } + offset += header.frameLength; + } + if (!firstHeader) { + return Promise.reject(new shaka.util.Error( + shaka.util.Error.Severity.CRITICAL, + shaka.util.Error.Category.MEDIA, + shaka.util.Error.Code.TRANSMUXING_FAILED)); + } + /** @type {number} */ + const sampleRate = firstHeader.sampleRate; + /** @type {number} */ + const frameDuration = + firstHeader.samplesPerFrame * timescale / firstHeader.sampleRate; + /** @type {number} */ + const baseMediaDecodeTime = this.frameIndex_ * frameDuration; + + /** @type {shaka.util.Mp4Generator.StreamInfo} */ + const streamInfo = { + timescale: sampleRate, + duration: duration, + videoNalus: [], + data: { + sequenceNumber: this.frameIndex_, + baseMediaDecodeTime: baseMediaDecodeTime, + samples: samples, + }, + stream: stream, + }; + const mp4Generator = new shaka.util.Mp4Generator(streamInfo); + if (!this.initSegment) { + this.initSegment = mp4Generator.initSegment(); + } + const segmentData = mp4Generator.segmentData(); + + this.frameIndex_++; + const transmuxData = Uint8ArrayUtils.concat(this.initSegment, segmentData); + return Promise.resolve(transmuxData); + } +}; + +shaka.transmuxer.TransmuxerEngine.registerTransmuxer( + 'audio/mpeg', + () => new shaka.transmuxer.Mp3Transmuxer('audio/mpeg'), + shaka.transmuxer.TransmuxerEngine.PluginPriority.FALLBACK); diff --git a/lib/transmuxer/mpeg_audio.js b/lib/transmuxer/mpeg_audio.js new file mode 100644 index 0000000000..bc169cd2b0 --- /dev/null +++ b/lib/transmuxer/mpeg_audio.js @@ -0,0 +1,196 @@ +/*! @license + * Shaka Player + * Copyright 2016 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +goog.provide('shaka.transmuxer.MpegAudio'); + + +/** + * MPEG parser utils + * + * @see https://en.wikipedia.org/wiki/MP3 + */ +shaka.transmuxer.MpegAudio = class { + /** + * @param {!Uint8Array} data + * @param {!number} offset + * @return {?{sampleRate: number, channelCount: number, + * frameLength: number, samplesPerFrame: number}} + */ + static parseHeader(data, offset) { + const MpegAudio = shaka.transmuxer.MpegAudio; + + const mpegVersion = (data[offset + 1] >> 3) & 3; + const mpegLayer = (data[offset + 1] >> 1) & 3; + const bitRateIndex = (data[offset + 2] >> 4) & 15; + const sampleRateIndex = (data[offset + 2] >> 2) & 3; + if (mpegVersion !== 1 && bitRateIndex !== 0 && + bitRateIndex !== 15 && sampleRateIndex !== 3) { + const paddingBit = (data[offset + 2] >> 1) & 1; + const channelMode = data[offset + 3] >> 6; + let columnInBitrates = 0; + if (mpegVersion === 3) { + columnInBitrates = 3 - mpegLayer; + } else { + columnInBitrates = mpegLayer === 3 ? 3 : 4; + } + const bitRate = MpegAudio.BITRATES_MAP_[ + columnInBitrates * 14 + bitRateIndex - 1] * 1000; + const columnInSampleRates = + mpegVersion === 3 ? 0 : mpegVersion === 2 ? 1 : 2; + const sampleRate = MpegAudio.SAMPLINGRATE_MAP_[ + columnInSampleRates * 3 + sampleRateIndex]; + // If bits of channel mode are `11` then it is a single channel (Mono) + const channelCount = channelMode === 3 ? 1 : 2; + const sampleCoefficient = + MpegAudio.SAMPLES_COEFFICIENTS_[mpegVersion][mpegLayer]; + const bytesInSlot = MpegAudio.BYTES_IN_SLOT_[mpegLayer]; + const samplesPerFrame = sampleCoefficient * 8 * bytesInSlot; + const frameLength = + Math.floor((sampleCoefficient * bitRate) / sampleRate + paddingBit) * + bytesInSlot; + + const userAgent = navigator.userAgent || ''; + // This affect to Tizen also for example. + const result = userAgent.match(/Chrome\/(\d+)/i); + const chromeVersion = result ? parseInt(result[1], 10) : 0; + const needChromeFix = !!chromeVersion && chromeVersion <= 87; + + if (needChromeFix && mpegLayer === 2 && + bitRate >= 224000 && channelMode === 0) { + // Work around bug in Chromium by setting channelMode + // to dual-channel (01) instead of stereo (00) + data[offset + 3] = data[offset + 3] | 0x80; + } + + return {sampleRate, channelCount, frameLength, samplesPerFrame}; + } + return null; + } + + /** + * @param {!Uint8Array} data + * @param {!number} offset + * @return {boolean} + */ + static isHeaderPattern(data, offset) { + return ( + data[offset] === 0xff && + (data[offset + 1] & 0xe0) === 0xe0 && + (data[offset + 1] & 0x06) !== 0x00 + ); + } + + /** + * @param {!Uint8Array} data + * @param {!number} offset + * @return {boolean} + */ + static isHeader(data, offset) { + // Look for MPEG header | 1111 1111 | 111X XYZX | where X can be either + // 0 or 1 and Y or Z should be 1 + // Layer bits (position 14 and 15) in header should be always different + // from 0 (Layer I or Layer II or Layer III) + // More info http://www.mp3-tech.org/programmer/frame_header.html + return offset + 1 < data.length && + shaka.transmuxer.MpegAudio.isHeaderPattern(data, offset); + } + + /** + * @param {!Uint8Array} data + * @param {!number} offset + * @return {boolean} + */ + static probe(data, offset) { + const MpegAudio = shaka.transmuxer.MpegAudio; + // same as isHeader but we also check that MPEG frame follows last + // MPEG frame or end of data is reached + if (offset + 1 < data.length && + MpegAudio.isHeaderPattern(data, offset)) { + // MPEG header Length + const headerLength = 4; + // MPEG frame Length + const header = MpegAudio.parseHeader(data, offset); + let frameLength = headerLength; + if (header && header.frameLength) { + frameLength = header.frameLength; + } + + const newOffset = offset + frameLength; + return newOffset === data.length || + MpegAudio.isHeader(data, newOffset); + } + return false; + } +}; + + +/** + * @private {!Array.} + */ +shaka.transmuxer.MpegAudio.BITRATES_MAP_ = [ + 32, 64, 96, 128, 160, 192, 224, 256, 288, 320, 352, 384, 416, 448, 32, 48, 56, + 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 384, 32, 40, 48, 56, 64, 80, + 96, 112, 128, 160, 192, 224, 256, 320, 32, 48, 56, 64, 80, 96, 112, 128, 144, + 160, 176, 192, 224, 256, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, + 160, +]; + +/** + * @private {!Array.} + */ +shaka.transmuxer.MpegAudio.SAMPLINGRATE_MAP_ = [ + 44100, 48000, 32000, 22050, 24000, 16000, 11025, 12000, 8000, +]; + +/** + * @private {!Array.>} + */ +shaka.transmuxer.MpegAudio.SAMPLES_COEFFICIENTS_ = [ + // MPEG 2.5 + [ + 0, // Reserved + 72, // Layer3 + 144, // Layer2 + 12, // Layer1 + ], + // Reserved + [ + 0, // Reserved + 0, // Layer3 + 0, // Layer2 + 0, // Layer1 + ], + // MPEG 2 + [ + 0, // Reserved + 72, // Layer3 + 144, // Layer2 + 12, // Layer1 + ], + // MPEG 1 + [ + 0, // Reserved + 144, // Layer3 + 144, // Layer2 + 12, // Layer1 + ], +]; + + +/** + * @private {!Array.} + */ +shaka.transmuxer.MpegAudio.BYTES_IN_SLOT_ = [ + 0, // Reserved + 1, // Layer3 + 1, // Layer2 + 4, // Layer1 +]; + +/** + * @const {number} + */ +shaka.transmuxer.MpegAudio.MPEG_AUDIO_SAMPLE_PER_FRAME = 1152; diff --git a/lib/util/id3_utils.js b/lib/util/id3_utils.js index 2878e11418..a17ddad1fe 100644 --- a/lib/util/id3_utils.js +++ b/lib/util/id3_utils.js @@ -307,6 +307,43 @@ shaka.util.Id3Utils = class { } return frames; } + + /** + * Returns any adjacent ID3 tags found in data starting at offset, as one + * block of data + * @param {Uint8Array} id3Data - The ID3 data containing one or more ID3 tags + * @param {number=} offset - The offset at which to start searching + * @return {!Uint8Array} + * @export + */ + static getID3Data(id3Data, offset = 0) { + const Id3Utils = shaka.util.Id3Utils; + const front = offset; + let length = 0; + + while (Id3Utils.isHeader_(id3Data, offset)) { + if ((id3Data[offset + 5] >> 6) & 1) { + // skip extended header + length += 10; + } + // skip past ID3 header + length += 10; + + const size = Id3Utils.readSize_(id3Data, offset + 6); + length += size; + + if (Id3Utils.isFooter_(id3Data, offset + 10)) { + // ID3 footer is 10 bytes + length += 10; + } + offset += length; + } + + if (length > 0) { + return id3Data.subarray(front, front + length); + } + return new Uint8Array([]); + } }; /** diff --git a/lib/util/mp4_generator.js b/lib/util/mp4_generator.js index 69dc8c4b85..9a1766a86a 100644 --- a/lib/util/mp4_generator.js +++ b/lib/util/mp4_generator.js @@ -716,10 +716,13 @@ shaka.util.Mp4Generator = class { segmentData() { const Mp4Generator = shaka.util.Mp4Generator; const movie = this.moof_(); - const length = Mp4Generator.FTYP_.byteLength + movie.byteLength; + const mdat = this.mdat_(); + const length = Mp4Generator.FTYP_.byteLength + movie.byteLength + + mdat.byteLength; const result = new Uint8Array(length); result.set(Mp4Generator.FTYP_); result.set(movie, Mp4Generator.FTYP_.byteLength); + result.set(mdat, Mp4Generator.FTYP_.byteLength + movie.byteLength); return result; } @@ -880,6 +883,21 @@ shaka.util.Mp4Generator = class { return Mp4Generator.box('trun', bytes); } + /** + * Generate a MDAT box + * + * @return {!Uint8Array} + * @private + */ + mdat_() { + const Mp4Generator = shaka.util.Mp4Generator; + let bytes = new Uint8Array(0); + for (const sample of this.samples_) { + bytes = shaka.util.Uint8ArrayUtils.concat(bytes, sample.data); + } + return Mp4Generator.box('mdat', bytes); + } + /** * @param {number} number @@ -1164,12 +1182,15 @@ shaka.util.Mp4Generator.Data; /** * @typedef {{ + * data: !Uint8Array, * size: number, * duration: number, * cts: number, * flags: !shaka.util.Mp4Generator.Mp4SampleFlags * }} * + * @property {!Uint8Array} data + * The sample data. * @property {number} size * The sample size. * @property {number} duration diff --git a/shaka-player.uncompiled.js b/shaka-player.uncompiled.js index 578ce4eb90..b8d0711339 100644 --- a/shaka-player.uncompiled.js +++ b/shaka-player.uncompiled.js @@ -65,6 +65,8 @@ goog.require('shaka.text.WebVttGenerator'); goog.require('shaka.cea.CeaDecoder'); goog.require('shaka.cea.Mp4CeaParser'); goog.require('shaka.cea.TsCeaParser'); +goog.require('shaka.transmuxer.Mp3Transmuxer'); +goog.require('shaka.transmuxer.MpegAudio'); goog.require('shaka.transmuxer.TransmuxerEngine'); goog.require('shaka.transmuxer.MssTransmuxer'); goog.require('shaka.transmuxer.MuxjsTransmuxer'); diff --git a/test/transmuxer/transmuxer_integration.js b/test/transmuxer/transmuxer_integration.js index bf49675c1b..5f1f2256a3 100644 --- a/test/transmuxer/transmuxer_integration.js +++ b/test/transmuxer/transmuxer_integration.js @@ -83,6 +83,26 @@ describe('Transmuxer Player', () => { await player.unload(); }); + it('raw MP3', async () => { + // eslint-disable-next-line max-len + const url = 'https://pl.streamingvideoprovider.com/mp3-playlist/playlist.m3u8'; + + await player.load(url, /* startTime= */ null, + /* mimeType= */ undefined); + video.play(); + expect(player.isLive()).toBe(false); + + // Wait for the video to start playback. If it takes longer than 10 + // seconds, fail the test. + await waiter.waitForMovementOrFailOnTimeout(video, 10); + + // Play for 15 seconds, but stop early if the video ends. If it takes + // longer than 45 seconds, fail the test. + await waiter.waitUntilPlayheadReachesOrFailOnTimeout(video, 15, 45); + + await player.unload(); + }); + it('muxed H.264+AAC in TS', async () => { // eslint-disable-next-line max-len const url = 'https://cf-sf-video.wmspanel.com/local/raw/BigBuckBunny_320x180.mp4/playlist.m3u8';