From ff2edc18cd6401f1058ea176e52c5da4104a8e5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Velad=20Galv=C3=A1n?= Date: Mon, 21 Aug 2023 18:00:07 +0200 Subject: [PATCH] fix(HLS): Fix external subtitles out of sync in HLS (#5491) Fixes https://github.com/shaka-project/shaka-player/issues/5458 Fixes https://github.com/shaka-project/shaka-player/issues/5443 Backported to v4.2.x --- externs/shaka/manifest.js | 6 +++++- externs/shaka/offline.js | 6 +++++- lib/dash/dash_parser.js | 1 + lib/hls/hls_parser.js | 1 + lib/media/media_source_engine.js | 11 ++++++----- lib/media/streaming_engine.js | 4 ++-- lib/offline/indexeddb/v1_storage_cell.js | 1 + lib/offline/indexeddb/v2_storage_cell.js | 1 + lib/offline/manifest_converter.js | 1 + lib/offline/storage.js | 1 + lib/player.js | 3 +++ lib/util/periods.js | 2 ++ test/media/adaptation_set_unit.js | 1 + test/media/media_source_engine_unit.js | 2 +- test/offline/manifest_convert_unit.js | 5 +++++ test/test/util/manifest_generator.js | 2 ++ test/test/util/offline_utils.js | 1 + test/test/util/streaming_engine_util.js | 3 +++ 18 files changed, 42 insertions(+), 10 deletions(-) diff --git a/externs/shaka/manifest.js b/externs/shaka/manifest.js index de5b124093..53ecd85e13 100644 --- a/externs/shaka/manifest.js +++ b/externs/shaka/manifest.js @@ -321,7 +321,8 @@ shaka.extern.FetchCryptoKeysFunction; * tilesLayout: (string|undefined), * matchedStreams: * (!Array.|!Array.| - * undefined) + * undefined), + * external: boolean * }} * * @description @@ -434,6 +435,9 @@ shaka.extern.FetchCryptoKeysFunction; * @property {(!Array.|!Array.| * undefined)} matchedStreams * The streams in all periods which match the stream. Used for Dash. + * @property {boolean} external + * Indicate if the stream was added externally. + * Eg: external text tracks. * * @exportDoc */ diff --git a/externs/shaka/offline.js b/externs/shaka/offline.js index 54a7534dad..39826d22d3 100644 --- a/externs/shaka/offline.js +++ b/externs/shaka/offline.js @@ -132,7 +132,8 @@ shaka.extern.ManifestDB; * audioSamplingRate: ?number, * spatialAudio: boolean, * closedCaptions: Map., - * tilesLayout: (string|undefined) + * tilesLayout: (string|undefined), + * external: boolean * }} * * @property {number} id @@ -193,6 +194,9 @@ shaka.extern.ManifestDB; * The value is a grid-item-dimension consisting of two positive decimal * integers in the format: column-x-row ('4x3'). It describes the arrangement * of Images in a Grid. The minimum valid LAYOUT is '1x1'. + * @property {boolean} external + * Indicate if the stream was added externally. + * Eg: external text tracks. */ shaka.extern.StreamDB; diff --git a/lib/dash/dash_parser.js b/lib/dash/dash_parser.js index a11b9eba91..f8647ac2e9 100644 --- a/lib/dash/dash_parser.js +++ b/lib/dash/dash_parser.js @@ -1337,6 +1337,7 @@ shaka.dash.DashParser = class { hdr, tilesLayout, matchedStreams: [], + external: false, }; } diff --git a/lib/hls/hls_parser.js b/lib/hls/hls_parser.js index 42f13f866a..3a5478036d 100644 --- a/lib/hls/hls_parser.js +++ b/lib/hls/hls_parser.js @@ -1757,6 +1757,7 @@ shaka.hls.HlsParser = class { closedCaptions, hdr: undefined, tilesLayout: undefined, + external: false, }; return { diff --git a/lib/media/media_source_engine.js b/lib/media/media_source_engine.js index da1ec1b727..44dce01046 100644 --- a/lib/media/media_source_engine.js +++ b/lib/media/media_source_engine.js @@ -354,7 +354,7 @@ shaka.media.MediaSourceEngine = class { let mimeType = shaka.util.MimeUtils.getFullType( stream.mimeType, stream.codecs); if (contentType == ContentType.TEXT) { - this.reinitText(mimeType, sequenceMode); + this.reinitText(mimeType, sequenceMode, stream.external); } else { if ((forceTransmuxTS || !MediaSource.isTypeSupported(mimeType)) && shaka.media.Transmuxer.isSupported(mimeType, contentType)) { @@ -383,13 +383,14 @@ shaka.media.MediaSourceEngine = class { * Reinitialize the TextEngine for a new text type. * @param {string} mimeType * @param {boolean} sequenceMode + * @param {boolean} external */ - reinitText(mimeType, sequenceMode) { + reinitText(mimeType, sequenceMode, external) { if (!this.textEngine_) { this.textEngine_ = new shaka.text.TextEngine(this.textDisplayer_); } this.textEngine_.initParser(mimeType, sequenceMode, - this.segmentRelativeVttTiming_); + external || this.segmentRelativeVttTiming_); } /** @@ -564,7 +565,7 @@ shaka.media.MediaSourceEngine = class { // the video stream, so textEngine may not have been initialized. if (!this.textEngine_) { this.reinitText(shaka.util.MimeUtils.CEA608_CLOSED_CAPTION_MIMETYPE, - this.sequenceMode_); + this.sequenceMode_, /* external= */ false); } if (transmuxedData.metadata) { @@ -593,7 +594,7 @@ shaka.media.MediaSourceEngine = class { } else if (hasClosedCaptions) { if (!this.textEngine_) { this.reinitText(shaka.util.MimeUtils.CEA608_CLOSED_CAPTION_MIMETYPE, - this.sequenceMode_); + this.sequenceMode_, /* external= */ false); } // If it is the init segment for closed captions, initialize the closed // caption parser. diff --git a/lib/media/streaming_engine.js b/lib/media/streaming_engine.js index 98a6715f6a..13fdb49f2f 100644 --- a/lib/media/streaming_engine.js +++ b/lib/media/streaming_engine.js @@ -240,7 +240,7 @@ shaka.media.StreamingEngine = class { const mimeType = shaka.util.MimeUtils.getFullType( stream.mimeType, stream.codecs); this.playerInterface_.mediaSourceEngine.reinitText( - mimeType, this.manifest_.sequenceMode); + mimeType, this.manifest_.sequenceMode, stream.external); const textDisplayer = this.playerInterface_.mediaSourceEngine.getTextDisplayer(); @@ -447,7 +447,7 @@ shaka.media.StreamingEngine = class { const fullMimeType = shaka.util.MimeUtils.getFullType( stream.mimeType, stream.codecs); this.playerInterface_.mediaSourceEngine.reinitText( - fullMimeType, this.manifest_.sequenceMode); + fullMimeType, this.manifest_.sequenceMode, stream.external); } // Releases the segmentIndex of the old stream. diff --git a/lib/offline/indexeddb/v1_storage_cell.js b/lib/offline/indexeddb/v1_storage_cell.js index 3b17603617..ed6793ccd5 100644 --- a/lib/offline/indexeddb/v1_storage_cell.js +++ b/lib/offline/indexeddb/v1_storage_cell.js @@ -176,6 +176,7 @@ shaka.offline.indexeddb.V1StorageCell = class spatialAudio: false, closedCaptions: null, tilesLayout: undefined, + external: false, }; } diff --git a/lib/offline/indexeddb/v2_storage_cell.js b/lib/offline/indexeddb/v2_storage_cell.js index 33d9645911..2ca2b901b8 100644 --- a/lib/offline/indexeddb/v2_storage_cell.js +++ b/lib/offline/indexeddb/v2_storage_cell.js @@ -125,6 +125,7 @@ shaka.offline.indexeddb.V2StorageCell = class spatialAudio: false, closedCaptions: null, tilesLayout: undefined, + external: false, }; } diff --git a/lib/offline/manifest_converter.js b/lib/offline/manifest_converter.js index 14ebd48b90..2acf8177a6 100644 --- a/lib/offline/manifest_converter.js +++ b/lib/offline/manifest_converter.js @@ -202,6 +202,7 @@ shaka.offline.ManifestConverter = class { spatialAudio: streamDB.spatialAudio, closedCaptions: streamDB.closedCaptions, tilesLayout: streamDB.tilesLayout, + external: streamDB.external, }; return stream; diff --git a/lib/offline/storage.js b/lib/offline/storage.js index 2a507fc9ae..a383b28a94 100644 --- a/lib/offline/storage.js +++ b/lib/offline/storage.js @@ -1324,6 +1324,7 @@ shaka.offline.Storage = class { spatialAudio: stream.spatialAudio, closedCaptions: stream.closedCaptions, tilesLayout: stream.tilesLayout, + external: stream.external, }; const startTime = diff --git a/lib/player.js b/lib/player.js index def7703378..3ef108c52e 100644 --- a/lib/player.js +++ b/lib/player.js @@ -2233,6 +2233,7 @@ shaka.Player = class extends shaka.util.FakeEventTarget { audioSamplingRate: null, spatialAudio: false, closedCaptions: null, + external: false, }, bandwidth: 100, allowedByApplication: true, @@ -4627,6 +4628,7 @@ shaka.Player = class extends shaka.util.FakeEventTarget { audioSamplingRate: null, spatialAudio: false, closedCaptions: null, + external: true, }; const fullMimeType = shaka.util.MimeUtils.getFullType( @@ -5057,6 +5059,7 @@ shaka.Player = class extends shaka.util.FakeEventTarget { audioSamplingRate: null, spatialAudio: false, closedCaptions: null, + external: false, }; manifest.textStreams.push(textStream); closedCaptionsSet.add(id); diff --git a/lib/util/periods.js b/lib/util/periods.js index 4d9ba851b2..d058af6e15 100644 --- a/lib/util/periods.js +++ b/lib/util/periods.js @@ -1491,6 +1491,7 @@ shaka.util.PeriodCombiner = class { audioSamplingRate: null, spatialAudio: false, closedCaptions: null, + external: false, }; } @@ -1527,6 +1528,7 @@ shaka.util.PeriodCombiner = class { audioSamplingRate: null, spatialAudio: false, closedCaptions: null, + external: false, }; } diff --git a/test/media/adaptation_set_unit.js b/test/media/adaptation_set_unit.js index f6256da31c..511f7a8b38 100644 --- a/test/media/adaptation_set_unit.js +++ b/test/media/adaptation_set_unit.js @@ -218,6 +218,7 @@ describe('AdaptationSet', () => { forced: false, trickModeVideo: null, type: '', + external: false, }; } }); diff --git a/test/media/media_source_engine_unit.js b/test/media/media_source_engine_unit.js index 8a36609758..fd1e1ac326 100644 --- a/test/media/media_source_engine_unit.js +++ b/test/media/media_source_engine_unit.js @@ -1258,7 +1258,7 @@ describe('MediaSourceEngine', () => { }); it('destroys text engines', async () => { - mediaSourceEngine.reinitText('text/vtt', false); + mediaSourceEngine.reinitText('text/vtt', false, false); await mediaSourceEngine.destroy(); expect(mockTextEngine).toBeTruthy(); diff --git a/test/offline/manifest_convert_unit.js b/test/offline/manifest_convert_unit.js index aad5f1cdc5..724dbe97b2 100644 --- a/test/offline/manifest_convert_unit.js +++ b/test/offline/manifest_convert_unit.js @@ -316,6 +316,7 @@ describe('ManifestConverter', () => { audioSamplingRate: null, spatialAudio: false, closedCaptions: null, + external: false, }; return streamDB; @@ -389,6 +390,7 @@ describe('ManifestConverter', () => { spatialAudio: false, closedCaptions: null, tilesLayout: undefined, + external: false, }; } @@ -438,6 +440,7 @@ describe('ManifestConverter', () => { spatialAudio: false, closedCaptions: null, tilesLayout: undefined, + external: false, }; } @@ -486,6 +489,7 @@ describe('ManifestConverter', () => { spatialAudio: false, closedCaptions: null, tilesLayout: undefined, + external: false, }; } @@ -531,6 +535,7 @@ describe('ManifestConverter', () => { spatialAudio: streamDb.spatialAudio, closedCaptions: streamDb.closedCaptions, tilesLayout: streamDb.tilesLayout, + external: streamDb.external, }; expect(stream).toEqual(expectedStream); diff --git a/test/test/util/manifest_generator.js b/test/test/util/manifest_generator.js index 3357d01c66..7e2cefcc4f 100644 --- a/test/test/util/manifest_generator.js +++ b/test/test/util/manifest_generator.js @@ -544,6 +544,8 @@ shaka.test.ManifestGenerator.Stream = class { this.hdr = undefined; /** @type {(string|undefined)} */ this.tilesLayout = undefined; + /** @type {boolean} */ + this.external = false; } /** @type {shaka.extern.Stream} */ diff --git a/test/test/util/offline_utils.js b/test/test/util/offline_utils.js index 7ca655e848..d7ea48c3a9 100644 --- a/test/test/util/offline_utils.js +++ b/test/test/util/offline_utils.js @@ -56,6 +56,7 @@ shaka.test.OfflineUtils = class { audioSamplingRate: null, spatialAudio: false, closedCaptions: null, + external: false, }; } diff --git a/test/test/util/streaming_engine_util.js b/test/test/util/streaming_engine_util.js index 95ec24066f..021b97ab7d 100644 --- a/test/test/util/streaming_engine_util.js +++ b/test/test/util/streaming_engine_util.js @@ -410,6 +410,7 @@ shaka.test.StreamingEngineUtil = class { roles: [], forced: false, spatialAudio: false, + external: false, }; } @@ -447,6 +448,7 @@ shaka.test.StreamingEngineUtil = class { roles: [], forced: false, spatialAudio: false, + external: false, }; } @@ -482,6 +484,7 @@ shaka.test.StreamingEngineUtil = class { roles: [], forced: false, spatialAudio: false, + external: false, }; } };