From 7fc6052729d02db913c391c4a9ef23cfd8f24043 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Velad=20Galv=C3=A1n?= Date: Mon, 21 Aug 2023 18:00:07 +0200 Subject: [PATCH] fix(HLS): Fix external subtitles out of sync in HLS (#5491) Fixes https://github.com/shaka-project/shaka-player/issues/5458 Fixes https://github.com/shaka-project/shaka-player/issues/5443 --- externs/shaka/manifest.js | 6 +++++- externs/shaka/offline.js | 6 +++++- lib/dash/dash_parser.js | 1 + lib/hls/hls_parser.js | 1 + lib/media/media_source_engine.js | 9 +++++---- lib/media/streaming_engine.js | 4 ++-- lib/mss/mss_parser.js | 1 + lib/offline/indexeddb/v1_storage_cell.js | 1 + lib/offline/indexeddb/v2_storage_cell.js | 1 + lib/offline/manifest_converter.js | 1 + lib/offline/storage.js | 1 + lib/player.js | 4 ++++ lib/util/periods.js | 2 ++ test/media/adaptation_set_unit.js | 1 + test/media/media_source_engine_unit.js | 2 +- test/offline/manifest_convert_unit.js | 5 +++++ test/test/util/manifest_generator.js | 2 ++ test/test/util/offline_utils.js | 1 + test/test/util/streaming_engine_util.js | 3 +++ 19 files changed, 43 insertions(+), 9 deletions(-) diff --git a/externs/shaka/manifest.js b/externs/shaka/manifest.js index af879b79bd..8dc805cdf1 100644 --- a/externs/shaka/manifest.js +++ b/externs/shaka/manifest.js @@ -388,7 +388,8 @@ shaka.extern.FetchCryptoKeysFunction; * matchedStreams: * (!Array.|!Array.| * undefined), - * mssPrivateData: (shaka.extern.MssPrivateData|undefined) + * mssPrivateData: (shaka.extern.MssPrivateData|undefined), + * external: boolean * }} * * @description @@ -510,6 +511,9 @@ shaka.extern.FetchCryptoKeysFunction; * @property {(shaka.extern.MssPrivateData|undefined)} mssPrivateData * Microsoft Smooth Streaming only.
* Private MSS data that is necessary to be able to do transmuxing. + * @property {boolean} external + * Indicate if the stream was added externally. + * Eg: external text tracks. * * @exportDoc */ diff --git a/externs/shaka/offline.js b/externs/shaka/offline.js index 4296dea087..2c61b14f43 100644 --- a/externs/shaka/offline.js +++ b/externs/shaka/offline.js @@ -137,7 +137,8 @@ shaka.extern.ManifestDB; * audioSamplingRate: ?number, * spatialAudio: boolean, * closedCaptions: Map., - * tilesLayout: (string|undefined) + * tilesLayout: (string|undefined), + * external: boolean * }} * * @property {number} id @@ -200,6 +201,9 @@ shaka.extern.ManifestDB; * The value is a grid-item-dimension consisting of two positive decimal * integers in the format: column-x-row ('4x3'). It describes the arrangement * of Images in a Grid. The minimum valid LAYOUT is '1x1'. + * @property {boolean} external + * Indicate if the stream was added externally. + * Eg: external text tracks. */ shaka.extern.StreamDB; diff --git a/lib/dash/dash_parser.js b/lib/dash/dash_parser.js index 81df0dc42e..e35c06db8c 100644 --- a/lib/dash/dash_parser.js +++ b/lib/dash/dash_parser.js @@ -1397,6 +1397,7 @@ shaka.dash.DashParser = class { tilesLayout, matchedStreams: [], accessibilityPurpose, + external: false, }; } diff --git a/lib/hls/hls_parser.js b/lib/hls/hls_parser.js index 5954c6bcfb..20262ad1dc 100644 --- a/lib/hls/hls_parser.js +++ b/lib/hls/hls_parser.js @@ -2487,6 +2487,7 @@ shaka.hls.HlsParser = class { hdr: undefined, tilesLayout: undefined, accessibilityPurpose: null, + external: false, }; } diff --git a/lib/media/media_source_engine.js b/lib/media/media_source_engine.js index a3ecdef626..2161ddfe0c 100644 --- a/lib/media/media_source_engine.js +++ b/lib/media/media_source_engine.js @@ -414,7 +414,7 @@ shaka.media.MediaSourceEngine = class { let mimeType = shaka.util.MimeUtils.getFullType( stream.mimeType, stream.codecs); if (contentType == ContentType.TEXT) { - this.reinitText(mimeType, this.sequenceMode_); + this.reinitText(mimeType, this.sequenceMode_, stream.external); } else { let needTransmux = this.config_.forceTransmux; if (!shaka.media.Capabilities.isTypeSupported(mimeType) || @@ -486,13 +486,14 @@ shaka.media.MediaSourceEngine = class { * Reinitialize the TextEngine for a new text type. * @param {string} mimeType * @param {boolean} sequenceMode + * @param {boolean} external */ - reinitText(mimeType, sequenceMode) { + reinitText(mimeType, sequenceMode, external) { if (!this.textEngine_) { this.textEngine_ = new shaka.text.TextEngine(this.textDisplayer_); } this.textEngine_.initParser(mimeType, sequenceMode, - this.segmentRelativeVttTiming_); + external || this.segmentRelativeVttTiming_); } /** @@ -797,7 +798,7 @@ shaka.media.MediaSourceEngine = class { if (hasClosedCaptions && contentType == ContentType.VIDEO) { if (!this.textEngine_) { this.reinitText(shaka.util.MimeUtils.CEA608_CLOSED_CAPTION_MIMETYPE, - this.sequenceMode_); + this.sequenceMode_, /* external= */ false); } if (!this.captionParser_) { const basicType = mimeType.split(';', 1)[0]; diff --git a/lib/media/streaming_engine.js b/lib/media/streaming_engine.js index 56fefab2bb..5403610ec6 100644 --- a/lib/media/streaming_engine.js +++ b/lib/media/streaming_engine.js @@ -260,7 +260,7 @@ shaka.media.StreamingEngine = class { const mimeType = shaka.util.MimeUtils.getFullType( stream.mimeType, stream.codecs); this.playerInterface_.mediaSourceEngine.reinitText( - mimeType, this.manifest_.sequenceMode); + mimeType, this.manifest_.sequenceMode, stream.external); const textDisplayer = this.playerInterface_.mediaSourceEngine.getTextDisplayer(); @@ -478,7 +478,7 @@ shaka.media.StreamingEngine = class { const fullMimeType = shaka.util.MimeUtils.getFullType( stream.mimeType, stream.codecs); this.playerInterface_.mediaSourceEngine.reinitText( - fullMimeType, this.manifest_.sequenceMode); + fullMimeType, this.manifest_.sequenceMode, stream.external); } // Releases the segmentIndex of the old stream. diff --git a/lib/mss/mss_parser.js b/lib/mss/mss_parser.js index b643dbd773..e1e1bac047 100644 --- a/lib/mss/mss_parser.js +++ b/lib/mss/mss_parser.js @@ -536,6 +536,7 @@ shaka.mss.MssParser = class { codecPrivateData: null, }, accessibilityPurpose: null, + external: false, }; // This is specifically for text tracks. diff --git a/lib/offline/indexeddb/v1_storage_cell.js b/lib/offline/indexeddb/v1_storage_cell.js index f8893f5d40..b55737bd3e 100644 --- a/lib/offline/indexeddb/v1_storage_cell.js +++ b/lib/offline/indexeddb/v1_storage_cell.js @@ -177,6 +177,7 @@ shaka.offline.indexeddb.V1StorageCell = class spatialAudio: false, closedCaptions: null, tilesLayout: undefined, + external: false, }; } diff --git a/lib/offline/indexeddb/v2_storage_cell.js b/lib/offline/indexeddb/v2_storage_cell.js index 47b201a649..e394e61fd2 100644 --- a/lib/offline/indexeddb/v2_storage_cell.js +++ b/lib/offline/indexeddb/v2_storage_cell.js @@ -126,6 +126,7 @@ shaka.offline.indexeddb.V2StorageCell = class spatialAudio: false, closedCaptions: null, tilesLayout: undefined, + external: false, }; } diff --git a/lib/offline/manifest_converter.js b/lib/offline/manifest_converter.js index dbce6e8e07..6acfdf7f45 100644 --- a/lib/offline/manifest_converter.js +++ b/lib/offline/manifest_converter.js @@ -208,6 +208,7 @@ shaka.offline.ManifestConverter = class { closedCaptions: streamDB.closedCaptions, tilesLayout: streamDB.tilesLayout, accessibilityPurpose: null, + external: streamDB.external, }; return stream; diff --git a/lib/offline/storage.js b/lib/offline/storage.js index 4e489257db..d1c02e5799 100644 --- a/lib/offline/storage.js +++ b/lib/offline/storage.js @@ -1323,6 +1323,7 @@ shaka.offline.Storage = class { spatialAudio: stream.spatialAudio, closedCaptions: stream.closedCaptions, tilesLayout: stream.tilesLayout, + external: stream.external, }; const startTime = diff --git a/lib/player.js b/lib/player.js index 0db95ed207..38aa6a7bab 100644 --- a/lib/player.js +++ b/lib/player.js @@ -2397,6 +2397,7 @@ shaka.Player = class extends shaka.util.FakeEventTarget { spatialAudio: false, closedCaptions: null, accessibilityPurpose: null, + external: false, }, bandwidth: 100, allowedByApplication: true, @@ -4889,6 +4890,7 @@ shaka.Player = class extends shaka.util.FakeEventTarget { spatialAudio: false, closedCaptions: null, accessibilityPurpose: null, + external: true, }; const fullMimeType = shaka.util.MimeUtils.getFullType( @@ -5042,6 +5044,7 @@ shaka.Player = class extends shaka.util.FakeEventTarget { closedCaptions: null, tilesLayout: '1x1', accessibilityPurpose: null, + external: true, }; this.manifest_.imageStreams.push(stream); @@ -5464,6 +5467,7 @@ shaka.Player = class extends shaka.util.FakeEventTarget { spatialAudio: false, closedCaptions: null, accessibilityPurpose: null, + external: false, }; manifest.textStreams.push(textStream); closedCaptionsSet.add(id); diff --git a/lib/util/periods.js b/lib/util/periods.js index ab0921cf63..a411b46042 100644 --- a/lib/util/periods.js +++ b/lib/util/periods.js @@ -1514,6 +1514,7 @@ shaka.util.PeriodCombiner = class { audioSamplingRate: null, spatialAudio: false, closedCaptions: null, + external: false, }; } @@ -1552,6 +1553,7 @@ shaka.util.PeriodCombiner = class { spatialAudio: false, closedCaptions: null, accessibilityPurpose: null, + external: false, }; } diff --git a/test/media/adaptation_set_unit.js b/test/media/adaptation_set_unit.js index af2960ec0e..be9c335ea6 100644 --- a/test/media/adaptation_set_unit.js +++ b/test/media/adaptation_set_unit.js @@ -220,6 +220,7 @@ describe('AdaptationSet', () => { trickModeVideo: null, type: '', accessibilityPurpose: null, + external: false, }; } }); diff --git a/test/media/media_source_engine_unit.js b/test/media/media_source_engine_unit.js index 94fdccc081..7019b156e9 100644 --- a/test/media/media_source_engine_unit.js +++ b/test/media/media_source_engine_unit.js @@ -1196,7 +1196,7 @@ describe('MediaSourceEngine', () => { }); it('destroys text engines', async () => { - mediaSourceEngine.reinitText('text/vtt', false); + mediaSourceEngine.reinitText('text/vtt', false, false); await mediaSourceEngine.destroy(); expect(mockTextEngine).toBeTruthy(); diff --git a/test/offline/manifest_convert_unit.js b/test/offline/manifest_convert_unit.js index dd25a53466..f693e751cd 100644 --- a/test/offline/manifest_convert_unit.js +++ b/test/offline/manifest_convert_unit.js @@ -317,6 +317,7 @@ describe('ManifestConverter', () => { audioSamplingRate: null, spatialAudio: false, closedCaptions: null, + external: false, }; return streamDB; @@ -392,6 +393,7 @@ describe('ManifestConverter', () => { closedCaptions: null, tilesLayout: undefined, accessibilityPurpose: null, + external: false, }; } @@ -443,6 +445,7 @@ describe('ManifestConverter', () => { closedCaptions: null, tilesLayout: undefined, accessibilityPurpose: null, + external: false, }; } @@ -493,6 +496,7 @@ describe('ManifestConverter', () => { closedCaptions: null, tilesLayout: undefined, accessibilityPurpose: null, + external: false, }; } @@ -540,6 +544,7 @@ describe('ManifestConverter', () => { closedCaptions: streamDb.closedCaptions, tilesLayout: streamDb.tilesLayout, accessibilityPurpose: null, + external: streamDb.external, }; expect(stream).toEqual(expectedStream); diff --git a/test/test/util/manifest_generator.js b/test/test/util/manifest_generator.js index 04238cdfcd..fa82eaae72 100644 --- a/test/test/util/manifest_generator.js +++ b/test/test/util/manifest_generator.js @@ -555,6 +555,8 @@ shaka.test.ManifestGenerator.Stream = class { this.tilesLayout = undefined; /** @type {?shaka.media.ManifestParser.AccessibilityPurpose} */ this.accessibilityPurpose; + /** @type {boolean} */ + this.external = false; } /** @type {shaka.extern.Stream} */ diff --git a/test/test/util/offline_utils.js b/test/test/util/offline_utils.js index 8106864607..90f10f8796 100644 --- a/test/test/util/offline_utils.js +++ b/test/test/util/offline_utils.js @@ -57,6 +57,7 @@ shaka.test.OfflineUtils = class { audioSamplingRate: null, spatialAudio: false, closedCaptions: null, + external: false, }; } diff --git a/test/test/util/streaming_engine_util.js b/test/test/util/streaming_engine_util.js index 6e9310a8b3..d9da79ff3f 100644 --- a/test/test/util/streaming_engine_util.js +++ b/test/test/util/streaming_engine_util.js @@ -415,6 +415,7 @@ shaka.test.StreamingEngineUtil = class { forced: false, spatialAudio: false, accessibilityPurpose: null, + external: false, }; } @@ -455,6 +456,7 @@ shaka.test.StreamingEngineUtil = class { forced: false, spatialAudio: false, accessibilityPurpose: null, + external: false, }; } @@ -492,6 +494,7 @@ shaka.test.StreamingEngineUtil = class { forced: false, spatialAudio: false, accessibilityPurpose: null, + external: false, }; } };