From cfcca8e383d2255dcd22a6f6a2463c9feeb0cd57 Mon Sep 17 00:00:00 2001 From: Ashwin Bhat Date: Tue, 21 Mar 2023 04:03:23 -0400 Subject: [PATCH] fix(HLS): support discontinuities in segments mode (#5102) --- lib/hls/hls_parser.js | 9 +- lib/media/media_source_engine.js | 4 - lib/media/segment_reference.js | 5 +- lib/media/streaming_engine.js | 18 +++- test/hls/hls_parser_unit.js | 112 +++++++++++++++++++++ test/test/util/fake_media_source_engine.js | 4 + 6 files changed, 141 insertions(+), 11 deletions(-) diff --git a/lib/hls/hls_parser.js b/lib/hls/hls_parser.js index 619bdadf21..543fd4e7aa 100644 --- a/lib/hls/hls_parser.js +++ b/lib/hls/hls_parser.js @@ -2371,13 +2371,15 @@ shaka.hls.HlsParser = class { * @param {!Map.} variables * @param {string} absoluteMediaPlaylistUri * @param {string} type + * @param {number} timestampOffset * @param {shaka.extern.HlsAes128Key=} hlsAes128Key * @return {shaka.media.SegmentReference} * @private */ createSegmentReference_( initSegmentReference, previousReference, hlsSegment, startTime, - variables, absoluteMediaPlaylistUri, type, hlsAes128Key) { + variables, absoluteMediaPlaylistUri, type, timestampOffset, + hlsAes128Key) { const tags = hlsSegment.tags; const absoluteSegmentUri = this.variableSubstitution_( hlsSegment.absoluteUri, variables); @@ -2547,7 +2549,7 @@ shaka.hls.HlsParser = class { startByte, endByte, initSegmentReference, - /* timestampOffset= */ 0, // This value is ignored in sequence mode. + timestampOffset, // This value is ignored in sequence mode. /* appendWindowStart= */ 0, /* appendWindowEnd= */ Infinity, partialSegmentRefs, @@ -2632,6 +2634,7 @@ shaka.hls.HlsParser = class { const references = []; let previousReference = null; + let lastDiscontinuityStartTime = firstStartTime; for (let i = 0; i < hlsSegments.length; i++) { const item = hlsSegments[i]; @@ -2643,6 +2646,7 @@ shaka.hls.HlsParser = class { item.tags, 'EXT-X-DISCONTINUITY'); if (discontinuityTag) { discontinuitySequence++; + lastDiscontinuityStartTime = startTime; } // Apply new AES-128 tags as you see them, keeping a running total. @@ -2681,6 +2685,7 @@ shaka.hls.HlsParser = class { variables, playlist.absoluteUri, type, + lastDiscontinuityStartTime, hlsAes128Key); previousReference = reference; diff --git a/lib/media/media_source_engine.js b/lib/media/media_source_engine.js index ced14a4160..7ca978ecb9 100644 --- a/lib/media/media_source_engine.js +++ b/lib/media/media_source_engine.js @@ -952,7 +952,6 @@ shaka.media.MediaSourceEngine = class { /** * Adjust timestamp offset to maintain AV sync across discontinuities. - * Only used in sequence mode. * * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @param {number} timestampOffset @@ -961,9 +960,6 @@ shaka.media.MediaSourceEngine = class { async resync(contentType, timestampOffset) { const ContentType = shaka.util.ManifestParserUtils.ContentType; - goog.asserts.assert(this.sequenceMode_, - 'resync only used with sequence mode!'); - if (contentType == ContentType.TEXT) { // This operation is for audio and video only. return; diff --git a/lib/media/segment_reference.js b/lib/media/segment_reference.js index cfde258c8f..796d5c62da 100644 --- a/lib/media/segment_reference.js +++ b/lib/media/segment_reference.js @@ -144,8 +144,9 @@ shaka.media.SegmentReference = class { * minus the first segment's tfdt box's 'baseMediaDecodeTime' field (after * it has been converted to seconds). *
- * For HLS, this value should be 0 to keep the presentation time at the most - * recent discontinuity minus the corresponding media time. + * For HLS, this value should be the start time of the most recent + * discontinuity, or 0 if there is no preceding discontinuity. Only used + * in segments mode. * @param {number} appendWindowStart * The start of the append window for this reference, relative to the * presentation. Any content from before this time will be removed by diff --git a/lib/media/streaming_engine.js b/lib/media/streaming_engine.js index b53fe892c6..f1cdbb111b 100644 --- a/lib/media/streaming_engine.js +++ b/lib/media/streaming_engine.js @@ -1675,19 +1675,31 @@ shaka.media.StreamingEngine = class { } } + const lastDiscontinuitySequence = + mediaState.lastSegmentReference ? + mediaState.lastSegmentReference.discontinuitySequence : null; if (this.manifest_.sequenceMode) { // Across discontinuity bounds, we should resync timestamps for // sequence mode playbacks. The next segment appended should // land at its theoretical timestamp from the segment index. - const lastDiscontinuitySequence = - mediaState.lastSegmentReference ? - mediaState.lastSegmentReference.discontinuitySequence : null; if (reference.discontinuitySequence != lastDiscontinuitySequence || mediaState.needsResync) { mediaState.needsResync = false; operations.push(this.playerInterface_.mediaSourceEngine.resync( mediaState.type, reference.startTime)); } + } else { + // In segments mode, we need to resync to set the timestampOffset + // to the start of the current discontinuity sequence. This is + // because individual discontinuity sequences may have internal + // timestamps that overlap, so we adjust the timestampOffset to avoid + // having the SourceBuffer get overwritten. + if (reference.discontinuitySequence != lastDiscontinuitySequence) { + operations.push( + this.playerInterface_.mediaSourceEngine.resync( + mediaState.type, + reference.timestampOffset)); + } } await Promise.all(operations); diff --git a/test/hls/hls_parser_unit.js b/test/hls/hls_parser_unit.js index efcd93abc3..8abfd9bdd8 100644 --- a/test/hls/hls_parser_unit.js +++ b/test/hls/hls_parser_unit.js @@ -987,6 +987,118 @@ describe('HlsParser', () => { await testHlsParser(master, media, manifest); }); + it('parses discontinuity tags', async () => { + const master = [ + '#EXTM3U\n', + '#EXT-X-STREAM-INF:BANDWIDTH=2000000,CODECS="avc1"\n', + 'video\n', + ].join(''); + + const media = [ + '#EXTM3U\n', + '#EXT-X-VERSION:3\n', + '#EXT-X-TARGETDURATION:5\n', + '#EXT-X-MEDIA-SEQUENCE:0\n', + '#EXTINF:3,\n', + 'clip0-video-0.ts\n', + '#EXTINF:1,\n', + 'clip0-video-1.ts\n', + '#EXT-X-DISCONTINUITY\n', + '#EXTINF:2,\n', + 'clip1-video-1.ts\n', + '#EXTINF:3,\n', + 'clip1-video-2.ts\n', + '#EXT-X-DISCONTINUITY\n', + '#EXTINF:1,\n', + 'media-clip2-video-0.ts\n', + '#EXTINF:1,\n', + 'media-clip2-video-1.ts\n', + '#EXT-X-DISCONTINUITY\n', + '#EXTINF:4,\n', + 'media-clip3-video-1.ts\n', + '#EXT-X-ENDLIST\n', + ].join(''); + + fakeNetEngine + .setResponseText('test:/master', master) + .setResponseText('test:/video', media); + + const manifest = await parser.start('test:/master', playerInterface); + await manifest.variants[0].video.createSegmentIndex(); + + const segmentIndex = manifest.variants[0].video.segmentIndex; + const references = []; + + for (let i = 0; i < 7; i++) { + references.push(segmentIndex.get(i)); + } + + expect(references[0].discontinuitySequence).toBe(0); + expect(references[1].discontinuitySequence).toBe(0); + expect(references[2].discontinuitySequence).toBe(1); + expect(references[3].discontinuitySequence).toBe(1); + expect(references[4].discontinuitySequence).toBe(2); + expect(references[5].discontinuitySequence).toBe(2); + expect(references[6].discontinuitySequence).toBe(3); + }); + + it('sets reference timetampOffset based on discontinuity start time', + async () => { + const master = [ + '#EXTM3U\n', + '#EXT-X-STREAM-INF:BANDWIDTH=2000000,CODECS="avc1"\n', + 'video\n', + ].join(''); + + const media = [ + '#EXTM3U\n', + '#EXT-X-VERSION:3\n', + '#EXT-X-TARGETDURATION:5\n', + '#EXT-X-MEDIA-SEQUENCE:0\n', + '#EXTINF:3,\n', + 'clip0-video-0.ts\n', + '#EXTINF:1,\n', + 'clip0-video-1.ts\n', + '#EXT-X-DISCONTINUITY\n', + '#EXTINF:2,\n', + 'clip1-video-1.ts\n', + '#EXTINF:3,\n', + 'clip1-video-2.ts\n', + '#EXT-X-DISCONTINUITY\n', + '#EXTINF:1,\n', + 'media-clip2-video-0.ts\n', + '#EXTINF:1,\n', + 'media-clip2-video-1.ts\n', + '#EXT-X-DISCONTINUITY\n', + '#EXTINF:4,\n', + 'media-clip3-video-1.ts\n', + '#EXT-X-ENDLIST\n', + ].join(''); + + fakeNetEngine + .setResponseText('test:/master', master) + .setResponseText('test:/video', media); + + const manifest = await parser.start('test:/master', playerInterface); + await manifest.variants[0].video.createSegmentIndex(); + + const segmentIndex = manifest.variants[0].video.segmentIndex; + const references = []; + + for (let i = 0; i < 7; i++) { + references.push(segmentIndex.get(i)); + } + + expect(references[0].timestampOffset).toBe(0); + expect(references[1].timestampOffset).toBe(0); + expect(references[2].timestampOffset).toBe(4); + expect(references[3].timestampOffset).toBe(4); + expect(references[4].timestampOffset).toBe(9); + expect(references[5].timestampOffset).toBe(9); + expect(references[6].timestampOffset).toBe(11); + }, + ); + it('parses characteristics from audio tags', async () => { const master = [ '#EXTM3U\n', diff --git a/test/test/util/fake_media_source_engine.js b/test/test/util/fake_media_source_engine.js index 40224f0943..97980861c9 100644 --- a/test/test/util/fake_media_source_engine.js +++ b/test/test/util/fake_media_source_engine.js @@ -136,6 +136,10 @@ shaka.test.FakeMediaSourceEngine = class { /** @type {!jasmine.Spy} */ this.updateLcevcDil = jasmine.createSpy('updateLcevcDil').and.stub(); + + /** @type {!jasmine.Spy} */ + this.resync= + jasmine.createSpy('resync').and.stub(); } /** @override */