Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Improve sequence mode start time #5326

Merged
merged 1 commit into from
Jun 20, 2023
Merged

feat: Improve sequence mode start time #5326

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 13 additions & 84 deletions lib/media/media_source_engine.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,9 +133,6 @@ shaka.media.MediaSourceEngine = class {
/** @private {boolean} */
this.ignoreManifestTimestampsInSegmentsMode_ = false;

/** @private {boolean} */
this.hasTextStreams_ = true;

/** @private {!shaka.util.PublicPromise.<number>} */
this.textSequenceModeOffset_ = new shaka.util.PublicPromise();
}
Expand Down Expand Up @@ -372,22 +369,19 @@ shaka.media.MediaSourceEngine = class {
* segment durations being out of sync with segment durations. In other
* words, assume that there are no gaps in the segments when appending
* to the SourceBuffer, even if the manifest and segment times disagree.
* @param {boolean=} hasTextStreams
* Indicates if the manifest has text streams.
*
* @return {!Promise}
*/
async init(streamsByType, sequenceMode=false,
manifestType=shaka.media.ManifestParser.UNKNOWN,
ignoreManifestTimestampsInSegmentsMode=false,
hasTextStreams=true) {
ignoreManifestTimestampsInSegmentsMode=false) {
await this.mediaSourceOpen_;

this.sequenceMode_ = sequenceMode;
this.manifestType_ = manifestType;
this.ignoreManifestTimestampsInSegmentsMode_ =
ignoreManifestTimestampsInSegmentsMode;
this.hasTextStreams_ = hasTextStreams;

for (const contentType of streamsByType.keys()) {
const stream = streamsByType.get(contentType);
Expand Down Expand Up @@ -454,9 +448,7 @@ shaka.media.MediaSourceEngine = class {
'expected \'open\'');
}

if (this.sequenceMode_ && !this.hasTextStreams_) {
// There's no text streams, so we can set sequence mode early instead
// of setting it after the first segment is appended in appendBuffer_.
if (this.sequenceMode_) {
sourceBuffer.mode =
shaka.media.MediaSourceEngine.SourceBufferMode_.SEQUENCE;
}
Expand Down Expand Up @@ -652,7 +644,7 @@ shaka.media.MediaSourceEngine = class {
* we are appending, or null for init segments
* @param {!string} mimeType
* @param {!number} timestampOffset
* @return {number}
* @return {?number}
* @private
*/
getTimestampAndDispatchMetadata_(contentType, data, reference, mimeType,
Expand Down Expand Up @@ -771,7 +763,7 @@ shaka.media.MediaSourceEngine = class {
}
const timestamp = this.getTimestampAndDispatchMetadata_(
contentType, data, reference, mimeType, timestampOffset);
if (attemptTimestampOffsetCalculation) {
if (timestamp != null && reference) {
const calculatedTimestampOffset = reference.startTime - timestamp;
const timestampOffsetDifference =
Math.abs(timestampOffset - calculatedTimestampOffset);
Expand All @@ -784,6 +776,15 @@ shaka.media.MediaSourceEngine = class {
() => this.setTimestampOffset_(contentType, timestampOffset));
}
}
// Timestamps can only be reliably extracted from video, not audio.
// Packed audio formats do not have internal timestamps at all.
// Prefer video for this when available.
const isBestSourceBufferForTimestamps =
contentType == ContentType.VIDEO ||
!(ContentType.VIDEO in this.sourceBuffers_);
if (this.sequenceMode_ && isBestSourceBufferForTimestamps) {
this.textSequenceModeOffset_.resolve(timestampOffset);
}
}
if (hasClosedCaptions && contentType == ContentType.VIDEO) {
if (!this.textEngine_) {
Expand Down Expand Up @@ -818,78 +819,6 @@ shaka.media.MediaSourceEngine = class {
data = this.workAroundBrokenPlatforms_(
data, reference ? reference.startTime : null, contentType);

const sourceBuffer = this.sourceBuffers_[contentType];
const SEQUENCE = shaka.media.MediaSourceEngine.SourceBufferMode_.SEQUENCE;

if (this.sequenceMode_ && sourceBuffer.mode != SEQUENCE && reference) {
// This is the first media segment to be appended to a SourceBuffer in
// sequence mode. We set the mode late so that we can trick MediaSource
// into extracting a timestamp for us to align text segments in sequence
// mode.

const duration = this.mediaSource_.duration;

// Timestamps can only be reliably extracted from video, not audio.
// Packed audio formats do not have internal timestamps at all.
// Prefer video for this when available.
const isBestSourceBufferForTimestamps =
contentType == ContentType.VIDEO ||
!(ContentType.VIDEO in this.sourceBuffers_);
if (isBestSourceBufferForTimestamps) {
// Append the segment in segments mode first, with offset of 0 and an
// open append window.
const originalRange =
[sourceBuffer.appendWindowStart, sourceBuffer.appendWindowEnd];
sourceBuffer.appendWindowStart = 0;
sourceBuffer.appendWindowEnd = Infinity;

const originalOffset = sourceBuffer.timestampOffset;
sourceBuffer.timestampOffset = 0;

await this.enqueueOperation_(
contentType, () => this.append_(contentType, data));
// If the input buffer passed to SourceBuffer#appendBuffer() does not
// contain a complete media segment, the call will exit while the
// SourceBuffer's append state is
// still PARSING_MEDIA_SEGMENT. Reset the parser state by calling
// abort() to safely reset timestampOffset to 'originalOffset'.
// https://www.w3.org/TR/media-source-2/#sourcebuffer-segment-parser-loop
await this.enqueueOperation_(
contentType, () => this.abort_(contentType));

// Reset the offset and append window.
sourceBuffer.timestampOffset = originalOffset;
sourceBuffer.appendWindowStart = originalRange[0];
sourceBuffer.appendWindowEnd = originalRange[1];

// Now get the timestamp of the segment and compute the offset for text
// segments.
const mediaStartTime = shaka.media.TimeRangesUtils.bufferStart(
this.getBuffered_(contentType));
const textOffset = (reference.startTime || 0) - (mediaStartTime || 0);
this.textSequenceModeOffset_.resolve(textOffset);

// Clear the buffer.
await this.enqueueOperation_(
contentType,
() => this.remove_(contentType, 0, duration));

// Finally, flush the buffer in case of choppy video start on HLS fMP4.
if (contentType == ContentType.VIDEO) {
await this.enqueueOperation_(
contentType,
() => this.flush_(contentType));
}
}

// Now switch to sequence mode and fall through to our normal operations.
sourceBuffer.mode = SEQUENCE;

// When we change the buffer mode the duration is lost, so we need to set
// it explicitly.
await this.setDuration(duration);
}

if (reference && this.sequenceMode_ && contentType != ContentType.TEXT) {
// In sequence mode, for non-text streams, if we just cleared the buffer
// and are either performing an unbuffered seek or handling an automatic
Expand Down
1 change: 0 additions & 1 deletion lib/media/streaming_engine.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -831,7 +831,6 @@ shaka.media.StreamingEngine = class {
this.manifest_.sequenceMode,
this.manifest_.type,
this.manifest_.ignoreManifestTimestampsInSegmentsMode,
this.manifest_.textStreams.length > 0,
);
this.destroyer_.ensureNotDestroyed();

Expand Down
54 changes: 0 additions & 54 deletions test/media/media_source_engine_unit.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -667,60 +667,6 @@ describe('MediaSourceEngine', () => {

expect(videoSourceBuffer.timestampOffset).toBe(0.50);
});

it('calls abort before setting timestampOffset', async () => {
const simulateUpdate = async () => {
await Util.shortDelay();
videoSourceBuffer.updateend();
};
const initObject = new Map();
initObject.set(ContentType.VIDEO, fakeVideoStream);

await mediaSourceEngine.init(initObject, /* sequenceMode= */ true);

// First, mock the scenario where timestampOffset is set to help align
// text segments. In this case, SourceBuffer mode is still 'segments'.
let reference = dummyReference(0, 1000);
let appendVideo = mediaSourceEngine.appendBuffer(
ContentType.VIDEO, buffer, reference, fakeStream,
/* hasClosedCaptions= */ false);
// Wait for the first appendBuffer(), in segments mode.
await simulateUpdate();
// Next, wait for abort(), used to reset the parser state for a safe
// setting of timestampOffset. Shaka fakes an updateend event on abort(),
// so simulateUpdate() isn't needed.
await Util.shortDelay();
// Next, wait for remove(), used to clear the SourceBuffer from the
// initial append.
await simulateUpdate();
// Next, wait for the second appendBuffer(), falling through to normal
// operations.
await simulateUpdate();
// Lastly, wait for the function-scoped MediaSourceEngine#appendBuffer()
// promise to resolve.
await appendVideo;
expect(videoSourceBuffer.abort).toHaveBeenCalledTimes(1);

// Second, mock the scenario where timestampOffset is set during an
// unbuffered seek or adaptation. SourceBuffer mode is 'sequence' now.
reference = dummyReference(0, 1000);
appendVideo = mediaSourceEngine.appendBuffer(
ContentType.VIDEO, buffer, reference, fakeStream,
/* hasClosedCaptions= */ false, /* seeked= */ true);
// First, wait for abort(), used to reset the parser state for a safe
// setting of timestampOffset.
await Util.shortDelay();
// The subsequent setTimestampOffset() fakes an updateend event for us, so
// simulateUpdate() isn't needed.
await Util.shortDelay();
// Next, wait for the second appendBuffer(), falling through to normal
// operations.
await simulateUpdate();
// Lastly, wait for the function-scoped MediaSourceEngine#appendBuffer()
// promise to resolve.
await appendVideo;
expect(videoSourceBuffer.abort).toHaveBeenCalledTimes(2);
});
});

describe('remove', () => {
Expand Down
3 changes: 1 addition & 2 deletions test/media/streaming_engine_unit.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -527,8 +527,7 @@ describe('StreamingEngine', () => {

expect(mediaSourceEngine.init).toHaveBeenCalledWith(expectedMseInit,
/** sequenceMode= */ false, /** manifestType= */ 'UNKNOWN',
/** ignoreManifestTimestampsInSegmentsMode= */ false,
/** hasTextStreams= */ true);
/** ignoreManifestTimestampsInSegmentsMode= */ false);
expect(mediaSourceEngine.init).toHaveBeenCalledTimes(1);

expect(mediaSourceEngine.setDuration).toHaveBeenCalledTimes(1);
Expand Down