diff --git a/CHANGELOG.md b/CHANGELOG.md index 8118d4640..ed0860f7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,16 @@ + +## [2.6.4](https://github.com/videojs/http-streaming/compare/v2.6.3...v2.6.4) (2021-03-12) + +### Bug Fixes + +* Monitor playback for stalls due to gaps in the beginning of stream when a new source is loaded ([#1087](https://github.com/videojs/http-streaming/issues/1087)) ([64a1f35](https://github.com/videojs/http-streaming/commit/64a1f35)) +* retry appends on QUOTA_EXCEEDED_ERR ([#1093](https://github.com/videojs/http-streaming/issues/1093)) ([008aeaf](https://github.com/videojs/http-streaming/commit/008aeaf)) + +### Chores + +* Get test coverage working again with mock/sync worker ([#1094](https://github.com/videojs/http-streaming/issues/1094)) ([035e8c0](https://github.com/videojs/http-streaming/commit/035e8c0)) +* pin CI to ubuntu 18.04 ([#1091](https://github.com/videojs/http-streaming/issues/1091)) ([01ca182](https://github.com/videojs/http-streaming/commit/01ca182)) + ## [2.6.3](https://github.com/videojs/http-streaming/compare/v2.6.2...v2.6.3) (2021-03-05) diff --git a/package-lock.json b/package-lock.json index 7249da0af..c247866c3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "@videojs/http-streaming", - "version": "2.6.3", + "version": "2.6.4", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 188414ada..67b179140 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@videojs/http-streaming", - "version": "2.6.3", + "version": "2.6.4", "description": "Play back HLS and DASH with Video.js, even where it's not natively supported", "main": "dist/videojs-http-streaming.cjs.js", "module": "dist/videojs-http-streaming.es.js", diff --git a/scripts/rollup.config.js b/scripts/rollup.config.js index e9b21bc83..1db948d0d 100644 --- a/scripts/rollup.config.js +++ b/scripts/rollup.config.js @@ -12,7 +12,6 @@ let syncWorker; const options = { input: 'src/videojs-http-streaming.js', distName: 'videojs-http-streaming', - checkWatch: false, excludeCoverage(defaults) { defaults.push(/^rollup-plugin-worker-factory/); defaults.push(/^create-test-data!/); diff --git a/src/error-codes.js b/src/error-codes.js new file mode 100644 index 000000000..241c0c3e7 --- /dev/null +++ b/src/error-codes.js @@ -0,0 +1,2 @@ +// https://www.w3.org/TR/WebIDL-1/#quotaexceedederror +export const QUOTA_EXCEEDED_ERR = 22; diff --git a/src/segment-loader.js b/src/segment-loader.js index b5903e5d3..490f3c7f8 100644 --- a/src/segment-loader.js +++ b/src/segment-loader.js @@ -21,6 +21,14 @@ import { } from './util/text-tracks'; import { gopsSafeToAlignWith, removeGopBuffer, updateGopBuffer } from './util/gops'; import shallowEqual from './util/shallow-equal.js'; +import { QUOTA_EXCEEDED_ERR } from './error-codes'; +import { timeRangesToArray } from './ranges'; + +// In the event of a quota exceeded error, keep at least one second of back buffer. This +// number was arbitrarily chosen and may be updated in the future, but seemed reasonable +// as a start to prevent any potential issues with removing content too close to the +// playhead. +const MIN_BACK_BUFFER = 1; // in ms const CHECK_BUFFER_DELAY = 500; @@ -509,6 +517,8 @@ export default class SegmentLoader extends videojs.EventTarget { id3: [], caption: [] }; + this.waitingOnRemove_ = false; + this.quotaExceededErrorRetryTimeout_ = null; // Fragmented mp4 playback this.activeInitSegmentId_ = null; @@ -692,6 +702,9 @@ export default class SegmentLoader extends videojs.EventTarget { this.metadataQueue_.id3 = []; this.metadataQueue_.caption = []; this.timelineChangeController_.clearPendingTimelineChange(this.loaderType_); + this.waitingOnRemove_ = false; + window.clearTimeout(this.quotaExceededErrorRetryTimeout_); + this.quotaExceededErrorRetryTimeout_ = null; } checkForAbort_(requestId) { @@ -1122,9 +1135,10 @@ export default class SegmentLoader extends videojs.EventTarget { * @param {number} start - the start time of the region to remove from the buffer * @param {number} end - the end time of the region to remove from the buffer * @param {Function} [done] - an optional callback to be executed when the remove + * @param {boolean} force - force all remove operations to happen * operation is complete */ - remove(start, end, done = () => {}) { + remove(start, end, done = () => {}, force = false) { // clamp end to duration if we need to remove everything. // This is due to a browser bug that causes issues if we remove to Infinity. // videojs/videojs-contrib-hls#1225 @@ -1147,7 +1161,7 @@ export default class SegmentLoader extends videojs.EventTarget { } }; - if (!this.audioDisabled_) { + if (force || !this.audioDisabled_) { removesRemaining++; this.sourceUpdater_.removeAudio(start, end, removeFinished); } @@ -1160,7 +1174,7 @@ export default class SegmentLoader extends videojs.EventTarget { // the event that we're switching between renditions and from video to audio only // (when we add support for that), we may need to clear the video contents despite // what the new media will contain. - if (this.loaderType_ === 'main') { + if (force || this.loaderType_ === 'main') { this.gopBuffer_ = removeGopBuffer(this.gopBuffer_, start, end, this.timeMapping_); removesRemaining++; this.sourceUpdater_.removeVideo(start, end, removeFinished); @@ -1893,6 +1907,12 @@ export default class SegmentLoader extends videojs.EventTarget { return false; } + // If content needs to be removed or the loader is waiting on an append reattempt, + // then no additional content should be appended until the prior append is resolved. + if (this.waitingOnRemove_ || this.quotaExceededErrorRetryTimeout_) { + return false; + } + const segmentInfo = this.pendingSegment_; // no segment to append any data for or @@ -2095,34 +2115,137 @@ export default class SegmentLoader extends videojs.EventTarget { return null; } - appendToSourceBuffer_({ segmentInfo, type, initSegment, data }) { - const segments = [data]; - let byteLength = data.byteLength; + handleQuotaExceededError_({segmentInfo, type, bytes}, error) { + const audioBuffered = this.sourceUpdater_.audioBuffered(); + const videoBuffered = this.sourceUpdater_.videoBuffered(); - if (initSegment) { - // if the media initialization segment is changing, append it before the content - // segment - segments.unshift(initSegment); - byteLength += initSegment.byteLength; + // For now we're ignoring any notion of gaps in the buffer, but they, in theory, + // should be cleared out during the buffer removals. However, log in case it helps + // debug. + if (audioBuffered.length > 1) { + this.logger_('On QUOTA_EXCEEDED_ERR, found gaps in the audio buffer: ' + + timeRangesToArray(audioBuffered).join(', ')); + } + if (videoBuffered.length > 1) { + this.logger_('On QUOTA_EXCEEDED_ERR, found gaps in the video buffer: ' + + timeRangesToArray(videoBuffered).join(', ')); } - // Technically we should be OK appending the init segment separately, however, we - // haven't yet tested that, and prepending is how we have always done things. - const bytes = concatSegments({ - bytes: byteLength, - segments - }); + const audioBufferStart = audioBuffered.length ? audioBuffered.start(0) : 0; + const audioBufferEnd = audioBuffered.length ? + audioBuffered.end(audioBuffered.length - 1) : 0; + const videoBufferStart = videoBuffered.length ? videoBuffered.start(0) : 0; + const videoBufferEnd = videoBuffered.length ? + videoBuffered.end(videoBuffered.length - 1) : 0; + + if ( + (audioBufferEnd - audioBufferStart) <= MIN_BACK_BUFFER && + (videoBufferEnd - videoBufferStart) <= MIN_BACK_BUFFER + ) { + // Can't remove enough buffer to make room for new segment (or the browser doesn't + // allow for appends of segments this size). In the future, it may be possible to + // split up the segment and append in pieces, but for now, error out this playlist + // in an attempt to switch to a more manageable rendition. + this.logger_('On QUOTA_EXCEEDED_ERR, single segment too large to append to ' + + 'buffer, triggering an error. ' + + `Appended byte length: ${bytes.byteLength}, ` + + `audio buffer: ${timeRangesToArray(audioBuffered).join(', ')}, ` + + `video buffer: ${timeRangesToArray(videoBuffered).join(', ')}, `); + this.error({ + message: 'Quota exceeded error with append of a single segment of content', + // To prevent any possible repeated downloads for content we can't actually + // append, blacklist forever. + blacklistDuration: Infinity + }); + this.trigger('error'); + return; + } + + // To try to resolve the quota exceeded error, clear back buffer and retry. This means + // that the segment-loader should block on future events until this one is handled, so + // that it doesn't keep moving onto further segments. Adding the call to the call + // queue will prevent further appends until waitingOnRemove_ and + // quotaExceededErrorRetryTimeout_ are cleared. + // + // Note that this will only block the current loader. In the case of demuxed content, + // the other load may keep filling as fast as possible. In practice, this should be + // OK, as it is a rare case when either audio has a high enough bitrate to fill up a + // source buffer, or video fills without enough room for audio to append (and without + // the availability of clearing out seconds of back buffer to make room for audio). + // But it might still be good to handle this case in the future as a TODO. + this.waitingOnRemove_ = true; + this.callQueue_.push(this.appendToSourceBuffer_.bind(this, {segmentInfo, type, bytes})); + + const currentTime = this.currentTime_(); + // Try to remove as much audio and video as possible to make room for new content + // before retrying. + const timeToRemoveUntil = currentTime - MIN_BACK_BUFFER; + + this.logger_(`On QUOTA_EXCEEDED_ERR, removing audio/video from 0 to ${timeToRemoveUntil}`); + this.remove(0, timeToRemoveUntil, () => { + + this.logger_(`On QUOTA_EXCEEDED_ERR, retrying append in ${MIN_BACK_BUFFER}s`); + this.waitingOnRemove_ = false; + // wait the length of time alotted in the back buffer to prevent wasted + // attempts (since we can't clear less than the minimum) + this.quotaExceededErrorRetryTimeout_ = window.setTimeout(() => { + this.logger_('On QUOTA_EXCEEDED_ERR, re-processing call queue'); + this.quotaExceededErrorRetryTimeout_ = null; + this.processCallQueue_(); + }, MIN_BACK_BUFFER * 1000); + }, true); + } - this.sourceUpdater_.appendBuffer({segmentInfo, type, bytes}, (error) => { - if (error) { - this.error(`${type} append of ${bytes.length}b failed for segment #${segmentInfo.mediaIndex} in playlist ${segmentInfo.playlist.id}`); - // If an append errors, we can't recover. - // (see https://w3c.github.io/media-source/#sourcebuffer-append-error). - // Trigger a special error so that it can be handled separately from normal, - // recoverable errors. - this.trigger('appenderror'); + handleAppendError_({segmentInfo, type, bytes}, error) { + // if there's no error, nothing to do + if (!error) { + return; + } + + if (error.code === QUOTA_EXCEEDED_ERR) { + this.handleQuotaExceededError_({segmentInfo, type, bytes}); + // A quota exceeded error should be recoverable with a future re-append, so no need + // to trigger an append error. + return; + } + + this.logger_('Received non QUOTA_EXCEEDED_ERR on append', error); + this.error(`${type} append of ${bytes.length}b failed for segment ` + + `#${segmentInfo.mediaIndex} in playlist ${segmentInfo.playlist.id}`); + + // If an append errors, we often can't recover. + // (see https://w3c.github.io/media-source/#sourcebuffer-append-error). + // + // Trigger a special error so that it can be handled separately from normal, + // recoverable errors. + this.trigger('appenderror'); + } + + appendToSourceBuffer_({ segmentInfo, type, initSegment, data, bytes }) { + // If this is a re-append, bytes were already created and don't need to be recreated + if (!bytes) { + const segments = [data]; + let byteLength = data.byteLength; + + if (initSegment) { + // if the media initialization segment is changing, append it before the content + // segment + segments.unshift(initSegment); + byteLength += initSegment.byteLength; } - }); + + // Technically we should be OK appending the init segment separately, however, we + // haven't yet tested that, and prepending is how we have always done things. + bytes = concatSegments({ + bytes: byteLength, + segments + }); + } + + this.sourceUpdater_.appendBuffer( + {segmentInfo, type, bytes}, + this.handleAppendError_.bind(this, {segmentInfo, type, bytes}) + ); } handleSegmentTimingInfo_(type, requestId, segmentTimingInfo) { diff --git a/src/source-updater.js b/src/source-updater.js index ec88f383c..85309227d 100644 --- a/src/source-updater.js +++ b/src/source-updater.js @@ -8,6 +8,7 @@ import { bufferIntersection } from './ranges.js'; import {getMimeForCodec} from '@videojs/vhs-utils/es/codecs.js'; import window from 'global/window'; import toTitleCase from './util/to-title-case.js'; +import { QUOTA_EXCEEDED_ERR } from './error-codes'; const bufferTypes = [ 'video', @@ -101,16 +102,22 @@ const shiftQueue = (type, sourceUpdater) => { } sourceUpdater.queue.splice(queueIndex, 1); + // Keep a record that this source buffer type is in use. + // + // The queue pending operation must be set before the action is performed in the event + // that the action results in a synchronous event that is acted upon. For instance, if + // an exception is thrown that can be handled, it's possible that new actions will be + // appended to an empty queue and immediately executed, but would not have the correct + // pending information if this property was set after the action was performed. + sourceUpdater.queuePending[type] = queueEntry; queueEntry.action(type, sourceUpdater); if (!queueEntry.doneFn) { // synchronous operation, process next entry + sourceUpdater.queuePending[type] = null; shiftQueue(type, sourceUpdater); return; } - - // asynchronous operation, so keep a record that this source buffer type is in use - sourceUpdater.queuePending[type] = queueEntry; }; const cleanupBuffer = (type, sourceUpdater) => { @@ -132,7 +139,7 @@ const inSourceBuffers = (mediaSource, sourceBuffer) => mediaSource && sourceBuff Array.prototype.indexOf.call(mediaSource.sourceBuffers, sourceBuffer) !== -1; const actions = { - appendBuffer: (bytes, segmentInfo) => (type, sourceUpdater) => { + appendBuffer: (bytes, segmentInfo, onError) => (type, sourceUpdater) => { const sourceBuffer = sourceUpdater[`${type}Buffer`]; // can't do anything if the media source / source buffer is null @@ -143,7 +150,15 @@ const actions = { sourceUpdater.logger_(`Appending segment ${segmentInfo.mediaIndex}'s ${bytes.length} bytes to ${type}Buffer`); - sourceBuffer.appendBuffer(bytes); + try { + sourceBuffer.appendBuffer(bytes); + } catch (e) { + sourceUpdater.logger_(`Error with code ${e.code} ` + + (e.code === QUOTA_EXCEEDED_ERR ? '(QUOTA_EXCEEDED_ERR) ' : '') + + `when appending segment ${segmentInfo.mediaIndex} to ${type}Buffer`); + sourceUpdater.queuePending[type] = null; + onError(e); + } }, remove: (start, end) => (type, sourceUpdater) => { const sourceBuffer = sourceUpdater[`${type}Buffer`]; @@ -155,7 +170,11 @@ const actions = { } sourceUpdater.logger_(`Removing ${start} to ${end} from ${type}Buffer`); - sourceBuffer.remove(start, end); + try { + sourceBuffer.remove(start, end); + } catch (e) { + sourceUpdater.logger_(`Remove ${start} to ${end} from ${type}Buffer failed`); + } }, timestampOffset: (offset) => (type, sourceUpdater) => { const sourceBuffer = sourceUpdater[`${type}Buffer`]; @@ -546,10 +565,16 @@ export default class SourceUpdater extends videojs.EventTarget { return; } + // In the case of certain errors, for instance, QUOTA_EXCEEDED_ERR, updateend will + // not be fired. This means that the queue will be blocked until the next action + // taken by the segment-loader. Provide a mechanism for segment-loader to handle + // these errors by calling the doneFn with the specific error. + const onError = doneFn; + pushQueue({ type, sourceUpdater: this, - action: actions.appendBuffer(bytes, segmentInfo || {mediaIndex: -1}), + action: actions.appendBuffer(bytes, segmentInfo || {mediaIndex: -1}, onError), doneFn, name: 'appendBuffer' }); diff --git a/test/segment-loader.test.js b/test/segment-loader.test.js index 0617e3fed..ba36f9d10 100644 --- a/test/segment-loader.test.js +++ b/test/segment-loader.test.js @@ -49,6 +49,7 @@ import { } from 'create-test-data!segments'; import sinon from 'sinon'; import { timeRangesEqual } from './custom-assertions.js'; +import { QUOTA_EXCEEDED_ERR } from '../src/error-codes'; /* TODO // noop addSegmentMetadataCue_ since most test segments dont have real timing information @@ -4150,6 +4151,159 @@ QUnit.module('SegmentLoader', function(hooks) { ); }); }); + + QUnit.test('QUOTA_EXCEEDED_ERR no loader error triggered', function(assert) { + return setupMediaSource(loader.mediaSource_, loader.sourceUpdater_).then(() => { + const playlist = playlistWithDuration(40); + + loader.playlist(playlist); + loader.load(); + this.clock.tick(1); + + // mock some buffer to prevent an error from not being able to clear any buffer + loader.sourceUpdater_.audioBuffered = () => videojs.createTimeRanges([0, 5]); + loader.sourceUpdater_.videoBuffered = () => videojs.createTimeRanges([0, 5]); + + loader.sourceUpdater_.appendBuffer = ({type, bytes}, callback) => { + callback({type: 'QUOTA_EXCEEDED_ERR', code: QUOTA_EXCEEDED_ERR}); + assert.notOk(loader.error_, 'no error triggered on loader'); + }; + + standardXHRResponse(this.requests.shift(), muxedSegment()); + }); + }); + + QUnit.test('QUOTA_EXCEEDED_ERR triggers error if no room for single segment', function(assert) { + return setupMediaSource(loader.mediaSource_, loader.sourceUpdater_).then(() => { + return new Promise((resolve, reject) => { + const playlist = playlistWithDuration(40); + + loader.playlist(playlist); + loader.load(); + this.clock.tick(1); + + // appenderrors are fatal, we don't want them in this case + loader.one('appenderror', reject); + loader.one('error', resolve); + + loader.sourceUpdater_.appendBuffer = ({type, bytes}, callback) => { + callback({type: 'QUOTA_EXCEEDED_ERR', code: QUOTA_EXCEEDED_ERR}); + }; + + standardXHRResponse(this.requests.shift(), muxedSegment()); + + }); + }).then(() => { + // buffer was empty, meaning there wasn't room for a single segment from that + // rendition + assert.deepEqual( + loader.error_, + { + message: 'Quota exceeded error with append of a single segment of content', + blacklistDuration: Infinity + }, + 'loader triggered and saved the error' + ); + }); + }); + + QUnit.test('QUOTA_EXCEEDED_ERR leads to clearing back buffer and retrying', function(assert) { + const removeVideoCalls = []; + const removeAudioCalls = []; + let origAppendBuffer; + + return setupMediaSource(loader.mediaSource_, loader.sourceUpdater_).then(() => { + const playlist = playlistWithDuration(40); + + loader.playlist(playlist); + loader.load(); + this.clock.tick(1); + + // mock some buffer and the playhead position + loader.currentTime_ = () => 7; + loader.sourceUpdater_.audioBuffered = () => videojs.createTimeRanges([2, 10]); + loader.sourceUpdater_.videoBuffered = () => videojs.createTimeRanges([0, 10]); + + loader.sourceUpdater_.removeVideo = (start, end, done) => { + assert.ok(loader.waitingOnRemove_, 'waiting on buffer removal to complete'); + removeVideoCalls.push({ start, end }); + done(); + }; + loader.sourceUpdater_.removeAudio = (start, end, done) => { + assert.ok(loader.waitingOnRemove_, 'waiting on buffer removal to complete'); + removeAudioCalls.push({ start, end }); + done(); + }; + + origAppendBuffer = loader.sourceUpdater_.appendBuffer; + loader.sourceUpdater_.appendBuffer = ({type, bytes}, callback) => { + assert.equal(removeVideoCalls.length, 0, 'no calls to remove video'); + assert.equal(removeAudioCalls.length, 0, 'no calls to remove audio'); + assert.notOk( + loader.waitingOnRemove_, + 'loader is not waiting on buffer removal' + ); + assert.notOk( + loader.quotaExceededErrorRetryTimeout_, + 'loader is not waiting to retry' + ); + assert.equal(loader.callQueue_.length, 0, 'loader has empty call queue'); + + callback({type: 'QUOTA_EXCEEDED_ERR', code: QUOTA_EXCEEDED_ERR}); + + assert.deepEqual( + removeVideoCalls, + [{ start: 0, end: 6 }], + 'removed video to one second behind playhead' + ); + assert.deepEqual( + removeAudioCalls, + [{ start: 0, end: 6 }], + 'removed audio to one second behind playhead' + ); + assert.notOk( + loader.waitingOnRemove_, + 'loader is not waiting on buffer removal' + ); + assert.ok( + loader.quotaExceededErrorRetryTimeout_, + 'loader is waiting to retry' + ); + assert.equal(loader.callQueue_.length, 1, 'loader has call waiting in queue'); + + loader.sourceUpdater_.appendBuffer = origAppendBuffer; + + // wait one second for retry timeout + this.clock.tick(1000); + + // ensure we cleared out the waiting state and call queue + assert.notOk( + loader.quotaExceededErrorRetryTimeout_, + 'loader is not waiting to retry' + ); + assert.equal(loader.callQueue_.length, 0, 'loader has empty call queue'); + }; + + standardXHRResponse(this.requests.shift(), muxedSegment()); + + return new Promise((resolve, reject) => { + loader.one('appended', resolve); + loader.one('error', reject); + }); + }).then(() => { + // at this point the append should've successfully completed, but it's good to + // once again check that the old state that was used was cleared out + assert.notOk( + loader.waitingOnRemove_, + 'loader is not waiting on buffer removal' + ); + assert.notOk( + loader.quotaExceededErrorRetryTimeout_, + 'loader is not waiting to retry' + ); + assert.equal(loader.callQueue_.length, 0, 'loader has empty call queue'); + }); + }); }); }); diff --git a/test/source-updater.test.js b/test/source-updater.test.js index cad857a73..4545f5454 100644 --- a/test/source-updater.test.js +++ b/test/source-updater.test.js @@ -5,6 +5,7 @@ import videojs from 'video.js'; import SourceUpdater from '../src/source-updater'; import {mp4VideoInit, mp4AudioInit, mp4Video, mp4Audio} from 'create-test-data!segments'; import { timeRangesEqual } from './custom-assertions.js'; +import { QUOTA_EXCEEDED_ERR } from '../src/error-codes'; const checkInitialDuration = function({duration}) { // ie sometimes sets duration to infinity earlier then expected @@ -1313,3 +1314,38 @@ QUnit[testOrSkip]('audio appends are delayed until video append for the first ap assert.ok(!audioAppend, 'audio has not appended yet'); }); }); + +QUnit.test('appendBuffer calls back with QUOTA_EXCEEDED_ERR', function(assert) { + assert.expect(2); + + this.sourceUpdater.createSourceBuffers({ + audio: 'mp4a.40.2', + video: 'avc1.4D001E' + }); + + const videoBuffer = { + appendBuffer() { + const quotaExceededError = new Error(); + + quotaExceededError.code = QUOTA_EXCEEDED_ERR; + + throw quotaExceededError; + } + }; + + const origMediaSource = this.sourceUpdater.mediaSource; + const origVideoBuffer = this.sourceUpdater.videoBuffer; + + // mock the media source and video buffer since you can't modify the native buffer + this.sourceUpdater.videoBuffer = videoBuffer; + this.sourceUpdater.mediaSource = { + sourceBuffers: [videoBuffer] + }; + + this.sourceUpdater.appendBuffer({type: 'video', bytes: mp4VideoTotal()}, (err) => { + assert.equal(err.code, QUOTA_EXCEEDED_ERR, 'called back with error'); + assert.notOk(this.sourceUpdater.queuePending.video, 'no pending action'); + this.sourceUpdater.mediaSource = origMediaSource; + this.sourceUpdater.videoBuffer = origVideoBuffer; + }); +});